notion-search 0.1.0__tar.gz

notion_search-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,18 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi        # the GitHub environment you created; PyPI trusts this exact name
+    permissions:
+      id-token: write        # lets PyPI verify this run without an API token (trusted publishing)
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv build
+      - uses: pypa/gh-action-pypi-publish@release/v1

notion_search-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+.DS_Store
+_old/
+DigUp/
+*.log
+dist/
+*.egg-info/

notion_search-0.1.0/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.5
+Name: notion-search
+Version: 0.1.0
+Summary: Offline, private hybrid search (exact words + meaning) for Notion Desktop's Cmd+K, running on your Mac
+Project-URL: Homepage, https://github.com/dipanshudaga/Notion-Search
+Project-URL: Issues, https://github.com/dipanshudaga/Notion-Search/issues
+Author: Dipanshu Daga
+Keywords: embeddings,mlx,notion,offline,search
+Classifier: Environment :: Console
+Classifier: Operating System :: MacOS
+Classifier: Programming Language :: Python :: 3
+Requires-Python: <3.14,>=3.12
+Requires-Dist: huggingface-hub>=1.33
+Requires-Dist: mlx-vlm>=0.7.8
+Requires-Dist: mlx>=0.32.3
+Requires-Dist: numpy>=2.5
+Requires-Dist: transformers>=5.19
+Requires-Dist: websockets>=13
+Description-Content-Type: text/markdown
+
+# Notion Search
+
+Notion's built-in search often misses what you are looking for. It leans on exact words and titles, it can skip text buried inside long pages, toggles and database rows, and its ranking is hard to predict.
+
+Notion Search keeps the same Cmd+K box and replaces what answers it. Every page in your workspace is indexed on your Mac, and each query is answered by a hybrid of exact-word search and meaning-based search. The meaning side uses Google's **EmbeddingGemma 2** model, running locally on the Apple Silicon GPU. Nothing leaves your machine, and it keeps working offline.
+
+## Install
+
+You need a Mac with Apple Silicon (M1 or newer) and Notion Desktop in `/Applications`. Open Terminal and run:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/dipanshudaga/Notion-Search/main/install.sh | bash
+```
+
+The first run takes 5 to 10 minutes. It installs the tool with `uv`, then works through three numbered steps:
+
+0. Installs `uv` if you do not have it. `uv` fetches Python 3.12 and the Notion Search package into their own isolated environment. Git, Homebrew and Xcode are not required.
+1. Downloads the EmbeddingGemma 2 model from Hugging Face (about 900 MB, once, no account needed). The audio part of the model, which this app does not use, is skipped. The model is stored in the app's own folder.
+2. Reads your pages from Notion's local database and builds the search index.
+3. Registers a background service that starts at every login.
+
+Plan for about 2 GB of free disk space. When the output says "Background service started", you are done.
+
+## Using it
+
+Open Notion and press **Cmd+K** (or Cmd+P). Nothing else changes.
+
+Notion has to be started with a debug port so the service can answer its search requests. If you open Notion normally, the service notices within a couple of seconds, quits it, and reopens it with the port enabled. Expect one short restart each time you launch Notion from the Dock or Spotlight. Your pages are safe, because Notion saves continuously.
+
+New and edited pages are picked up automatically, usually within 15 seconds. You never need to run an update command.
+
+## What gets searched
+
+- The full text of every page, however long, including text inside toggles, columns, lists and callouts.
+- Table cells.
+- Page titles, and the breadcrumb path of each page.
+
+Long pages are split into chunks of about 1,500 characters. Each chunk is embedded separately, so a match deep inside a long page still finds the page, and the result snippet shows the part that matched.
+
+Not indexed: database properties such as status, tags or dates, and the contents of PDFs, images and other attachments. Pages Notion Desktop has never loaded onto your Mac are not in its local database, so they cannot be indexed until Notion loads them.
+
+## How results are ranked
+
+Each query is run two ways and the results are merged.
+
+- **Exact words:** SQLite FTS5 with stemming and prefix matching. A title match counts more than a body match.
+- **Meaning:** the query is embedded and compared with every chunk. A chunk only counts when its similarity stands well above the typical score for that query (4 standard deviations), which keeps unrelated pages out and lets nonsense queries return nothing.
+
+Pages that contain all of your words are boosted, and titles that contain them are boosted again. The snippet shown under each result is cut from the place in the page where your rarest words cluster, with the matches highlighted.
+
+## The control menu
+
+Run `notion-search` in Terminal for an arrow-key menu:
+
+| Option | What it does |
+|---|---|
+| Update | Reinstalls the service and refreshes dependencies |
+| Status | Shows whether the service is running and Notion is hooked |
+| Restart | Restarts the background service |
+| Reindex | Rebuilds the whole index from scratch (about 2 to 3 minutes) |
+| Uninstall | Removes the service, optionally with the index and model |
+
+The same actions work as commands: `notion-search status`, `restart`, `reindex`, `uninstall [--purge]`, and `logs` for debugging. If the command is not found, run `uv tool update-shell` and open a new Terminal window.
+
+## Troubleshooting
+
+- **Results look slow or come from Notion's own search:** run `notion-search status`. The service must be running and Notion must be hooked.
+- **A new page does not appear:** wait about 15 seconds, and make sure Notion Desktop has opened that page at least once.
+- **Results look stale or wrong:** run Reindex from the menu.
+- **Something else:** `notion-search logs` follows `~/Library/Logs/NotionSearch.log`.
+
+## Uninstall
+
+Run `notion-search`, choose Uninstall, then **Service** to keep the index and model or **Everything** to also delete the index and the 900 MB model. To remove the program itself afterwards, run `uv tool uninstall notion-search`. Notion itself is never modified, so it behaves exactly as before.
+
+## Privacy and security
+
+- Everything runs locally. No page text, query or telemetry is sent anywhere. The only download is the model (about 900 MB), once.
+- Notion's database is opened read-only, and Notion's app files are never touched, so its code signature stays intact.
+- The service talks to Notion over Chromium's DevTools port (127.0.0.1:9222). That port is reachable by any process running as you on this Mac, so do not use this on a Mac that other people control.
+
+## How it works
+
+```
+Notion Desktop ── Cmd+K ──> /api/v3/search request
+        │  (intercepted over the DevTools port)
+        ▼
+notion_service.py ──> search_engine.py ──> answer in Notion's own response format
+        │                    │
+        │                    ├─ FTS5 index   (exact words)
+        │                    └─ vectors      (EmbeddingGemma 2, MLX, 8-bit)
+        ▼
+notion_indexer.py: reads Notion's local database (read-only),
+re-embeds only the pages whose text changed
+```
+
+| File | Role |
+|---|---|
+| `src/notion_search/cli.py` | The `notion-search` command: menu, install, status, reindex, uninstall |
+| `src/notion_search/service.py` | Background service: keeps Notion hooked, answers searches, keeps the index fresh |
+| `src/notion_search/engine.py` | Hybrid retrieval, ranking, snippets, and Notion's response format |
+| `src/notion_search/indexer.py` | Reads the Notion database, builds pages and chunks, stores the index |
+| `src/notion_search/embedder.py` | Runs EmbeddingGemma 2 on the GPU with the model card's prompts |
+| `install.sh` | One-line bootstrap: installs `uv`, then the package, then runs `notion-search install` |
+
+The index and the model live in `~/Library/Application Support/NotionSearch`. All model work runs on a single thread, because MLX streams are per-thread.
+
+## Performance
+
+Measured on an M2 MacBook Air with a 836-page workspace (956 chunks):
+
+- Search: about 20 to 50 ms per query, including the model call. The first query after a restart takes about 250 ms.
+- Memory: about 740 MB resident, most of it the model in 8-bit form (280 MB) and Python.
+- Full index build: 2 to 3 minutes. Later updates re-embed only changed pages and take about a second.
+- Idle: no measurable CPU, and no network traffic beyond local connections to Notion.
+
+## Limitations
+
+- Apple Silicon only (MLX).
+- Pages must be cached by Notion Desktop to be searchable.
+- Notion restarts once when it is opened without the debug port.
+- Database properties and attachment contents are not searched.
+
+## Credits
+
+- **[DigUp](https://github.com/ARahim3/DigUp)** by ARahim3 (MIT). DigUp, which searches local files with the same model, shaped several decisions here: the model card's query and document prompts, scoring a hit against the whole score distribution for a query instead of a fixed cutoff, storing a model fingerprint in the index so a model change triggers a rebuild, and the tested `mlx-vlm` version that supports the model.
+- **[EmbeddingGemma 2](https://huggingface.co/google/embeddinggemma-2)** by Google DeepMind (Apache 2.0).
+- **[MLX](https://github.com/ml-explore/mlx)** by Apple and **[mlx-vlm](https://github.com/Blaizzy/mlx-vlm)** by Blaizzy.
+- **SQLite FTS5** for exact-word search.
+
+Notion Search is not affiliated with Notion Labs, Google or Apple. Notion is a trademark of Notion Labs, Inc.

notion_search-0.1.0/README.md

@@ -0,0 +1,131 @@
+# Notion Search
+
+Notion's built-in search often misses what you are looking for. It leans on exact words and titles, it can skip text buried inside long pages, toggles and database rows, and its ranking is hard to predict.
+
+Notion Search keeps the same Cmd+K box and replaces what answers it. Every page in your workspace is indexed on your Mac, and each query is answered by a hybrid of exact-word search and meaning-based search. The meaning side uses Google's **EmbeddingGemma 2** model, running locally on the Apple Silicon GPU. Nothing leaves your machine, and it keeps working offline.
+
+## Install
+
+You need a Mac with Apple Silicon (M1 or newer) and Notion Desktop in `/Applications`. Open Terminal and run:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/dipanshudaga/Notion-Search/main/install.sh | bash
+```
+
+The first run takes 5 to 10 minutes. It installs the tool with `uv`, then works through three numbered steps:
+
+0. Installs `uv` if you do not have it. `uv` fetches Python 3.12 and the Notion Search package into their own isolated environment. Git, Homebrew and Xcode are not required.
+1. Downloads the EmbeddingGemma 2 model from Hugging Face (about 900 MB, once, no account needed). The audio part of the model, which this app does not use, is skipped. The model is stored in the app's own folder.
+2. Reads your pages from Notion's local database and builds the search index.
+3. Registers a background service that starts at every login.
+
+Plan for about 2 GB of free disk space. When the output says "Background service started", you are done.
+
+## Using it
+
+Open Notion and press **Cmd+K** (or Cmd+P). Nothing else changes.
+
+Notion has to be started with a debug port so the service can answer its search requests. If you open Notion normally, the service notices within a couple of seconds, quits it, and reopens it with the port enabled. Expect one short restart each time you launch Notion from the Dock or Spotlight. Your pages are safe, because Notion saves continuously.
+
+New and edited pages are picked up automatically, usually within 15 seconds. You never need to run an update command.
+
+## What gets searched
+
+- The full text of every page, however long, including text inside toggles, columns, lists and callouts.
+- Table cells.
+- Page titles, and the breadcrumb path of each page.
+
+Long pages are split into chunks of about 1,500 characters. Each chunk is embedded separately, so a match deep inside a long page still finds the page, and the result snippet shows the part that matched.
+
+Not indexed: database properties such as status, tags or dates, and the contents of PDFs, images and other attachments. Pages Notion Desktop has never loaded onto your Mac are not in its local database, so they cannot be indexed until Notion loads them.
+
+## How results are ranked
+
+Each query is run two ways and the results are merged.
+
+- **Exact words:** SQLite FTS5 with stemming and prefix matching. A title match counts more than a body match.
+- **Meaning:** the query is embedded and compared with every chunk. A chunk only counts when its similarity stands well above the typical score for that query (4 standard deviations), which keeps unrelated pages out and lets nonsense queries return nothing.
+
+Pages that contain all of your words are boosted, and titles that contain them are boosted again. The snippet shown under each result is cut from the place in the page where your rarest words cluster, with the matches highlighted.
+
+## The control menu
+
+Run `notion-search` in Terminal for an arrow-key menu:
+
+| Option | What it does |
+|---|---|
+| Update | Reinstalls the service and refreshes dependencies |
+| Status | Shows whether the service is running and Notion is hooked |
+| Restart | Restarts the background service |
+| Reindex | Rebuilds the whole index from scratch (about 2 to 3 minutes) |
+| Uninstall | Removes the service, optionally with the index and model |
+
+The same actions work as commands: `notion-search status`, `restart`, `reindex`, `uninstall [--purge]`, and `logs` for debugging. If the command is not found, run `uv tool update-shell` and open a new Terminal window.
+
+## Troubleshooting
+
+- **Results look slow or come from Notion's own search:** run `notion-search status`. The service must be running and Notion must be hooked.
+- **A new page does not appear:** wait about 15 seconds, and make sure Notion Desktop has opened that page at least once.
+- **Results look stale or wrong:** run Reindex from the menu.
+- **Something else:** `notion-search logs` follows `~/Library/Logs/NotionSearch.log`.
+
+## Uninstall
+
+Run `notion-search`, choose Uninstall, then **Service** to keep the index and model or **Everything** to also delete the index and the 900 MB model. To remove the program itself afterwards, run `uv tool uninstall notion-search`. Notion itself is never modified, so it behaves exactly as before.
+
+## Privacy and security
+
+- Everything runs locally. No page text, query or telemetry is sent anywhere. The only download is the model (about 900 MB), once.
+- Notion's database is opened read-only, and Notion's app files are never touched, so its code signature stays intact.
+- The service talks to Notion over Chromium's DevTools port (127.0.0.1:9222). That port is reachable by any process running as you on this Mac, so do not use this on a Mac that other people control.
+
+## How it works
+
+```
+Notion Desktop ── Cmd+K ──> /api/v3/search request
+        │  (intercepted over the DevTools port)
+        ▼
+notion_service.py ──> search_engine.py ──> answer in Notion's own response format
+        │                    │
+        │                    ├─ FTS5 index   (exact words)
+        │                    └─ vectors      (EmbeddingGemma 2, MLX, 8-bit)
+        ▼
+notion_indexer.py: reads Notion's local database (read-only),
+re-embeds only the pages whose text changed
+```
+
+| File | Role |
+|---|---|
+| `src/notion_search/cli.py` | The `notion-search` command: menu, install, status, reindex, uninstall |
+| `src/notion_search/service.py` | Background service: keeps Notion hooked, answers searches, keeps the index fresh |
+| `src/notion_search/engine.py` | Hybrid retrieval, ranking, snippets, and Notion's response format |
+| `src/notion_search/indexer.py` | Reads the Notion database, builds pages and chunks, stores the index |
+| `src/notion_search/embedder.py` | Runs EmbeddingGemma 2 on the GPU with the model card's prompts |
+| `install.sh` | One-line bootstrap: installs `uv`, then the package, then runs `notion-search install` |
+
+The index and the model live in `~/Library/Application Support/NotionSearch`. All model work runs on a single thread, because MLX streams are per-thread.
+
+## Performance
+
+Measured on an M2 MacBook Air with a 836-page workspace (956 chunks):
+
+- Search: about 20 to 50 ms per query, including the model call. The first query after a restart takes about 250 ms.
+- Memory: about 740 MB resident, most of it the model in 8-bit form (280 MB) and Python.
+- Full index build: 2 to 3 minutes. Later updates re-embed only changed pages and take about a second.
+- Idle: no measurable CPU, and no network traffic beyond local connections to Notion.
+
+## Limitations
+
+- Apple Silicon only (MLX).
+- Pages must be cached by Notion Desktop to be searchable.
+- Notion restarts once when it is opened without the debug port.
+- Database properties and attachment contents are not searched.
+
+## Credits
+
+- **[DigUp](https://github.com/ARahim3/DigUp)** by ARahim3 (MIT). DigUp, which searches local files with the same model, shaped several decisions here: the model card's query and document prompts, scoring a hit against the whole score distribution for a query instead of a fixed cutoff, storing a model fingerprint in the index so a model change triggers a rebuild, and the tested `mlx-vlm` version that supports the model.
+- **[EmbeddingGemma 2](https://huggingface.co/google/embeddinggemma-2)** by Google DeepMind (Apache 2.0).
+- **[MLX](https://github.com/ml-explore/mlx)** by Apple and **[mlx-vlm](https://github.com/Blaizzy/mlx-vlm)** by Blaizzy.
+- **SQLite FTS5** for exact-word search.
+
+Notion Search is not affiliated with Notion Labs, Google or Apple. Notion is a trademark of Notion Labs, Inc.

notion_search-0.1.0/context.md

@@ -0,0 +1,251 @@
+# Notion Edge AI Search: Project Context, Architecture & Findings
+
+## 1. Executive Summary & Vision
+This project replaces Notion Desktop's cloud search with a **100% offline, private, ultra-low latency hybrid search engine** powered by **Apple Silicon Metal GPU (MLX)** and **SQLite FTS5**.
+
+### Key Achievements:
+- **Zero UI Injection / Zero Custom Overlays:** Integrates directly into Notion Desktop's **official native Quick Find modal** (`Cmd+K` / `Cmd+P`).
+- **Instant Response Latency:** Reduced search response time from **6,186ms down to ~22ms** (a 250x speedup), outperforming cloud search.
+- **Factory Pristine Notion Binary:** Zero binary patching or `app.asar` modification. Notion runs with its official Apple Developer signature (`LBQJ96FQ8D`), eliminating macOS Keychain prompts completely.
+- **Ghost Page Purging:** Discovered and eliminated 297 orphaned database blocks (pages that could not be opened), bringing the active page index to 836 valid, live pages.
+- **Calibrated Semantic Relevance:** Fine-tuned semantic cosine similarity threshold (`0.68`) to eliminate hallucinations and irrelevant empty pages.
+
+---
+
+## 2. Technical Architecture
+
+```
+┌────────────────────────────────────────────────────────┐
+│                   Notion Desktop                       │
+│    (/Applications/Notion.app --remote-debugging-port)  │
+│                                                        │
+│  Native Quick Find Modal (React / SpaceStore)          │
+│       │                                                │
+│       ▼                                                │
+│  Fetch /api/v3/search                                  │
+└───────┬────────────────────────────────────────────────┘
+        │
+        │ [Chromium DevTools Protocol - CDP Interception]
+        ▼
+┌────────────────────────────────────────────────────────┐
+│              notion_service.py (Daemon)                │
+│                                                        │
+│  • Listens on ws://127.0.0.1:9222/devtools/page/...    │
+│  • Low-level CDP Fetch.enable for *api/v3/search*      │
+│  • Intercepts search request body                      │
+│  • Returns Fetch.fulfillRequest with v3 Schema JSON    │
+└───────┬────────────────────────────────────────────────┘
+        │
+        ▼
+┌────────────────────────────────────────────────────────┐
+│              search_daemon.py (SearchEngine)           │
+│                                                        │
+│  1. In-Memory Record Cache (RAM):                      │
+│     • 10,300 active blocks, 57 collections, spaces     │
+│     • Zero disk I/O, zero SQLite locks                 │
+│                                                        │
+│  2. Hybrid Retrieval Pipeline:                         │
+│     • Keyword: SQLite FTS5 (BM25, ~2ms)                │
+│     • Semantic: EmbeddingGemma-2 on MLX Metal GPU (~20ms)│
+│     • Reciprocal Rank Fusion (RRF) with 0.68 cutoff    │
+│     • Native highlight markup: <gzkNfoUU>...</gzkNfoUU>│
+└───────┬────────────────────────────────────────────────┘
+        │
+        ▼
+┌────────────────────────────────────────────────────────┐
+│            Local Storage / Sync Engine                 │
+│                                                        │
+│  • notion_index.db: FTS5 tables & vector storage       │
+│  • vectors.npy: 836 x 768 pre-computed embeddings      │
+│  • metadata.json: Document titles, paths, snippets     │
+│  • notion_indexer.py: Auto-sync on notion.db-wal edits │
+└────────────────────────────────────────────────────────┘
+```
+
+---
+
+## 3. Comprehensive Chronological Findings & Evolution
+
+### Phase 1: The Custom UI Overlay & Keybinding Conflicts
+- **Approach:** Initially injected a floating React-style overlay inside Notion via `injected_search.js` listening for `Cmd+K`.
+- **Issues Encountered:**
+  - Notion's native search modal operates on both `Cmd+K` and `Cmd+P`. The injected overlay intercepted `Cmd+K`, breaking Notion's native shortcut experience and feeling unnatural.
+  - Results looked like a separate web popup rather than feeling like Notion itself.
+- **Resolution:** Completely scrapped all custom UI overlays and event listeners. Decided to integrate exclusively with Notion's native search modal by intercepting the network protocol.
+
+---
+
+### Phase 2: Reverse Engineering Notion's Native `/api/v3/search` Schema
+- **Challenge:** When intercepting `/api/v3/search` via CDP, Notion's search modal displayed infinite loading skeletons and threw a runtime crash:
+  ```text
+  TypeError: Cannot convert undefined or null to object
+      at entries (<anonymous>)
+      at Object.current (https://app.notion.com/_assets/12650-57c02c4b54977214.js)
+  ```
+- **Reverse Engineering Findings:**
+  1. **`trackEventProperties`:** Notion calls `Object.entries(data.trackEventProperties)`. Passing `null` or omitting it crashes the React renderer. Must provide `trackEventProperties: {}`.
+  2. **`spaceId` & `table`:** Each item in `results` requires `spaceId` and `table: "block"`.
+  3. **`recordMap` Hydration:** Notion's React `SpaceStore` does not query the cloud for search result titles if they are present in `recordMap`. It requires:
+     - `recordMap.block`: Block data including `id`, `type`, `properties.title`, `format.page_icon`, `parent_id`, `parent_table`.
+     - `recordMap.space`: Workspace identity (the workspace name and ID read from the local `space` table).
+     - `recordMap.collection`: Parent database metadata for database row pages.
+  4. **Native Search Highlighting:**
+     - Notion does **not** use HTML `<b>` or `<span>` for search term highlighting. Using `<b>` renders raw text `<b>word</b>` on screen.
+     - Notion's internal text parser uses proprietary delimiter tags:
+       - Open: `<gzkNfoUU>`
+       - Close: `</gzkNfoUU>`
+     - When wrapped in `<gzkNfoUU>word</gzkNfoUU>`, Notion's native React components render the word with authentic **blue highlight styling**.
+
+---
+
+### Phase 3: The Keychain Prompt & Code Signing (Eliminated Forever)
+- **Problem:** macOS repeatedly displayed a prompt: `"Notion wants access to key 'Notion Safe Storage' in your keychain"`.
+- **Root Cause:**
+  - Earlier injection methods unpacked `/Applications/Notion.app/Contents/Resources/app.asar`, patched JavaScript files, and re-signed the application using ad-hoc codesigning (`codesign --sign -`).
+  - Chromium uses macOS Keychain to encrypt session cookies under `"Notion Safe Storage"`.
+  - The macOS Keychain Access Control List (ACL) explicitly matches the application's Code Signing Team ID.
+  - When the signature changed from Notion Labs' official certificate (`LBQJ96FQ8D`) to an ad-hoc local signature, macOS detected a binary tamper and blocked access until prompted.
+- **Permanent Fix:**
+  - Restored `/Applications/Notion.app` from the official Notion 7.37.1 distribution (`Developer ID Application: Notion Labs, Incorporated (LBQJ96FQ8D)`).
+  - **Rule Established:** Never touch, unpack, or modify Notion files on disk.
+  - Connect solely over Chromium DevTools Protocol (CDP) via `--remote-debugging-port=9222`.
+  - Because Notion's binary and code signature remain 100% authentic and untouched, macOS Keychain will **never prompt again**.
+
+---
+
+### Phase 4: Purging Ghost & Orphaned Pages
+- **Problem:** Searching for certain queries returned orphaned entries (pages that could not be opened) that could not be opened, showed no parent breadcrumbs, and could not be found anywhere inside Notion.
+- **Database Forensics:**
+  - Inspected Notion's local database (`~/Library/Application Support/Notion/notion.db`).
+  - Discovered 297 leftover database rows in the `block` table originating from an old imported and deleted CSV table (`List_of_Sahitya_Akademi_Award_winners_for_Hindi_1.csv`).
+  - While Notion marked the root collection as deleted, individual row blocks remained in the SQLite database with `alive = 1`, but their parent chain terminated prematurely without reaching a live workspace `space`.
+- **Resolution:**
+  - Rewrote the traversal filter in `notion_indexer.py` (`is_block_in_trash`):
+    ```python
+    def is_block_in_trash(block_id, block_meta, collection_meta, valid_spaces):
+        # Recursively walks parent chain: must reach an active space
+        # Any dead parent, trash timestamp, or disconnected link marks the page as trashed
+    ```
+  - Re-indexed the workspace using Apple Silicon MLX GPU (`GemmaEmbedder` accelerated via `mlx_vlm` in 82.81s).
+  - Cleaned index from 1,044 -> **836 strictly valid, live pages**. All ghost pages permanently purged.
+
+---
+
+### Phase 5: Semantic Threshold Calibration
+- **Problem:** Searching for terms like `"nomad"` previously matched loose, irrelevant documents (e.g. generic travel itineraries or unpopulated templates).
+- **Embedding Space Analysis:**
+  - 768-dimensional embedding distribution for Gemma-2:
+    - `0.48 – 0.58`: Baseline noise floor (unrelated notes).
+    - `0.60 – 0.66`: Weak semantic associations.
+    - `≥ 0.68`: High human relevance confidence (≥80% match).
+- **Resolution:**
+  - Added `MIN_SEMANTIC_THRESHOLD = 0.68` in `search_daemon.py`.
+  - Now, searching `"nomad"` returns strictly relevant notes (*Travel*, *Stays*, *Philosophy*) with full breadcrumbs (`Build / Startups`).
+
+---
+
+### Phase 6: Eliminating Search Latency (From 6,186ms to 22.9ms)
+- **Problem:** User reported: *"results are taking time to load"*. Logs revealed search queries were taking **5,105ms, 5,610ms, and 6,186ms**.
+- **Root Cause Diagnosis:**
+  - On every keystroke in Notion's search modal, `search_notion_v3_format` was opening a fresh connection to `~/Library/Application Support/Notion/notion.db` and executing multiple `PRAGMA table_info` and `SELECT * FROM block` queries in a loop.
+  - Notion Desktop actively writes transactions and WAL checkpoints to `notion.db` while running.
+  - When SQLite files are locked by the active application, Python's `sqlite3` driver waits up to its default busy timeout: **exactly 5.0 seconds (5000ms)**!
+  - Once that 5-second lock released, the query finally finished, causing noticeable lag.
+- **The Solution (In-Memory Record Cache):**
+  - Notion's 10,300 blocks, 57 collections, and workspace records total only **~5.8 MB of RAM**.
+  - In `search_daemon.py`, implemented `load_notion_record_cache()` to pre-load all block schemas, collections, and spaces into memory at startup.
+  - Zero disk I/O or SQLite queries occur during search execution.
+  - **Latency Benchmarks:**
+    | Query | Before (SQLite Disk Queries) | After (In-Memory Cache) | Speedup |
+    | :--- | :--- | :--- | :--- |
+    | `q="camera"` | 4,700 ms | **22.96 ms** | ~205x |
+    | `q="trip"` | 5,100 ms | **22.55 ms** | ~226x |
+    | `q="travel"` | 5,600 ms | **24.54 ms** | ~228x |
+    | `q="guide"` | 6,186 ms | **28.40 ms** | ~218x |
+
+---
+
+## 4. Exact Text of Latest Status Update (Preserved Verbatim; the manage.sh/LaunchAgent question is answered in section 5)
+
+```markdown
+Here is the direct diagnosis and resolution for each of your three points:
+
+---
+
+### 1. Why results were taking time to load (and the fix)
+
+#### **The Root Cause:**
+When analyzing `service.log`, queries were hitting delays of **5,105ms – 6,186ms**. 
+
+On every search keystroke, the service was connecting directly to Notion's active local database file (`~/Library/Application Support/Notion/notion.db`) to look up block, collection, and workspace schemas. Because Notion Desktop actively writes transactions and WAL checkpoints to that database while running, SQLite file locks occurred—forcing Python's `sqlite3` driver to hit its default **5.0-second busy timeout**.
+
+#### **The Fix:**
+- We updated `search_daemon.py` to **pre-cache all 10,300 active blocks, 57 collections, and workspace records directly into RAM** at startup (~5.8 MB total).
+- Search queries now run **100% in-memory** with local FTS5 and Apple Silicon Metal GPU (MLX).
+- **Benchmark comparison:**
+  | Metric | Before (Direct SQLite Queries) | After (In-Memory Pre-Cache) |
+  | :--- | :--- | :--- |
+  | `q="camera"` | ~4,700 ms | **22.9 ms** ⚡ |
+  | `q="trip"` | ~5,100 ms | **22.5 ms** ⚡ |
+  | `q="travel"` | ~5,600 ms | **24.5 ms** ⚡ |
+  | `q="guide"` | ~6,186 ms | **28.4 ms** ⚡ |
+
+Search results in Notion's native modal are now sub-30ms instant.
+
+---
+
+### 2. "Will it still require the Keychain access? Honestly I don't want that"
+
+**No. It will NEVER ask for Keychain access again.**
+
+#### **Why did it ask before?**
+Earlier iterations modified Notion's internal `app.asar` archive and re-signed the app with a temporary ad-hoc signature (`codesign -s -`). macOS Keychain protects "Notion Safe Storage" (Chromium's cookie encryption key) using Code Signing Access Control Lists. Because the signing identity changed from Notion Labs to an unsigned local identity, macOS security displayed the Keychain prompt.
+
+#### **Why is it gone for good now?**
+1. We completely restored `/Applications/Notion.app` from Notion's official release (`Developer ID Application: Notion Labs, Incorporated (LBQJ96FQ8D)`).
+2. We **never touch or patch Notion's files on disk**.
+3. We connect purely over the native Chromium DevTools Protocol (CDP) on localhost port 9222.
+4. Because the official binary is 100% intact and signed by Notion Labs, macOS Keychain recognizes the legitimate developer certificate and **will never prompt you again**.
+
+---
+
+### 3. Where are we at with the "Brew Install" / Distribution?
+
+Yes, it can be implemented. Here is how the distribution models work for this architecture:
+
+#### **What is required under the hood?**
+For this setup to work seamlessly without touching terminal commands:
+1. A background daemon running the search service on login.
+2. Notion launching with `--remote-debugging-port=9222`.
+
+#### **Comparison of Options:**
+
+| Approach | How it works | Pros | Cons |
+| :--- | :--- | :--- | :--- |
+| **Option A: 1-Click Native Installer (`install.sh`)** *(Recommended)* | Copies a `com.notion.edge-search.plist` into `~/Library/LaunchAgents/` and creates a transparent wrapper/shortcut for Notion. | • Standard approach for AI Mac utilities (e.g. Ollama)<br>• Zero external brew tap maintenance<br>• Self-contained in your workspace<br>• Launches silently on Mac boot | Requires running `./install.sh` once |
+| **Option B: Homebrew Tap (`brew install ...`)** | We create a GitHub repo with a Homebrew formula that downloads the code and runs `brew services start notion-search`. | • Familiar `brew install` CLI command | • Homebrew formulas struggle with heavy ML packages (`mlx`, `torch`, `transformers`) unless bundled as huge pre-built bottles.<br>• Still ultimately creates the same `~/Library/LaunchAgents` file behind the scenes. |
+
+### Recommendation:
+If you want a zero-friction experience right now, we can create an installer script that:
+1. Registers the service as a macOS **`LaunchAgent`** (so it starts automatically in the background when you turn on your Mac).
+2. Creates an application shortcut (or dock icon) so clicking Notion always opens it with debugging enabled.
+
+Would you like me to set up the automatic **LaunchAgent background service** so it runs seamlessly on boot without needing `./manage.sh`?
+```
+
+---
+
+## 5. Oct 15 rewrite (current state; sections 1-4 are history)
+
+What changed and why:
+- **One thread for MLX.** MLX streams are per-thread; the old code embedded from whatever `asyncio.to_thread` worker was free, and the sync watcher reloaded everything on the same pool. `service.log` showed 119 ms - 8.4 s per query, not 22 ms. Engine + SQLite now live on a single-thread executor. Measured after: ~25-40 ms per query through Notion.
+- **One index path.** `notion_indexer.snapshot()` + `store()` replace separate full/incremental code. Pages are re-embedded only when their text signature changes (moves/renames included). No more `vectors.npy` / `metadata.json` copies: vectors live in `index.db` and in RAM.
+- **Borrowed from DigUp:** the model card's prompts (`task: search result | query:` / `title: | text:`), a per-query z-score gate instead of the fixed 0.68 cosine cutoff (Z_GATE=4.0; real queries peak at z 4.6-5.5, gibberish 3.5, so nonsense now returns nothing), an index fingerprint (change prompts/model -> auto re-embed), single persistent embedding thread.
+- **Fixed on the way:** text segments were joined with spaces ("hel lo"); full index pulled child-page text into parents; teamspace pages were treated as trash; hardcoded space id / model path; page snippets re-embedded at 256 tokens (now 512).
+- **Gone:** injected overlay (`injected_search.js`), HTTP server, patch/unpatch scripts, `app.asar.original`, torch fallback (Apple Silicon only now), `manage.sh` (-> `install.sh`).
+- **Install:** `./install.sh` builds a venv, indexes, writes `~/Library/LaunchAgents/com.notionsearch.service.plist` (RunAtLoad + KeepAlive). Service restarts Notion once if it was opened without `--remote-debugging-port=9222` (untested on the live app as of this note).
+- Old versions of the rewritten files are in `_old/` (safe to delete).
+
+## 6. File Inventory
+
+See README.md.

notion_search-0.1.0/install.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Notion Search installer. Run it with this one command:
+#   curl -fsSL https://raw.githubusercontent.com/dipanshudaga/Notion-Search/main/install.sh | bash
+# Needs only a Mac with Apple Silicon: uv (installed here if missing) fetches Python 3.12 and the package.
+set -euo pipefail
+
+[[ "$(uname -s)-$(uname -m)" == "Darwin-arm64" ]] || { echo "Notion Search needs a Mac with an Apple Silicon chip."; exit 1; }
+
+export PATH="$HOME/.local/bin:$PATH"
+command -v uv >/dev/null || curl -LsSf https://astral.sh/uv/install.sh | UV_NO_MODIFY_PATH=1 sh >/dev/null
+
+# PyPI first; until the package is published there, fall back to the GitHub source archive (no Git needed).
+uv tool install --python 3.12 --force --quiet notion-search 2>/dev/null \
+  || uv tool install --python 3.12 --force --quiet https://github.com/dipanshudaga/Notion-Search/archive/refs/heads/main.tar.gz
+
+exec notion-search install

notion_search-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "notion-search"
+version = "0.1.0"
+description = "Offline, private hybrid search (exact words + meaning) for Notion Desktop's Cmd+K, running on your Mac"
+readme = "README.md"
+requires-python = ">=3.12,<3.14"
+authors = [{ name = "Dipanshu Daga" }]
+keywords = ["notion", "search", "embeddings", "mlx", "offline"]
+classifiers = ["Operating System :: MacOS", "Environment :: Console", "Programming Language :: Python :: 3"]
+dependencies = [
+    "mlx>=0.32.3",
+    "mlx-vlm>=0.7.8",            # EmbeddingGemma 2 support ships on PyPI from 0.7.x
+    "transformers>=5.19",
+    "huggingface-hub>=1.33",
+    "numpy>=2.5",
+    "websockets>=13",
+]
+
+[project.scripts]
+notion-search = "notion_search.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/dipanshudaga/Notion-Search"
+Issues = "https://github.com/dipanshudaga/Notion-Search/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/notion_search"]