lanternkb 0.1.0__tar.gz

lanternkb-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,75 @@
+# Publishes lanternkb and lanternkb-ui to PyPI when a version tag (v0.1.0) is pushed (decision 0028).
+# Trusted publishing: no token is stored anywhere. PyPI trusts this workflow in the `pypi` environment for lanternkb
+# and in `pypi-ui` for lanternkb-ui: PyPI takes one pending publisher per repository, workflow and environment, so the
+# two projects need two environments. Each is set up once on pypi.org (decision 0028, "Setting up PyPI").
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:  # a dry run: build and check, never publish (`gh workflow run release.yml`)
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7  # v10.2.0
+      - name: The tag matches every version
+        if: github.ref_type == 'tag'
+        run: |
+          v="${GITHUB_REF_NAME#v}"
+          grep -qx "version = \"$v\"" pyproject.toml
+          grep -qx "version = \"$v\"" packages/lanternkb-ui/pyproject.toml
+          grep -qx "__version__ = \"$v\"" src/lanternkb/__init__.py
+          grep -q "\"lanternkb-ui==$v\"" pyproject.toml
+          grep -q "\"lanternkb==$v\"" packages/lanternkb-ui/pyproject.toml
+      - name: The wheel ships no evaluation harness and nothing gitleaks flags (issue #28)
+        run: uv run --no-project --python 3.12 python scripts/check_wheel.py  # standard library only
+      - run: uv build --package lanternkb --out-dir dist/lanternkb
+      - run: uv build --package lanternkb-ui --out-dir dist/lanternkb-ui
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: lanternkb
+          path: dist/lanternkb/
+          if-no-files-found: error
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: lanternkb-ui
+          path: dist/lanternkb-ui/
+          if-no-files-found: error
+
+  publish-lanternkb:
+    needs: build
+    if: github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: pypi
+    permissions:
+      id-token: write  # trusted publishing
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        with:
+          name: lanternkb
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2
+
+  # After lanternkb, which it pins: never a settings page on PyPI without the version it needs.
+  publish-lanternkb-ui:
+    needs: publish-lanternkb
+    if: github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: pypi-ui
+    permissions:
+      id-token: write  # trusted publishing
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        with:
+          name: lanternkb-ui
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2

lanternkb-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,95 @@
+# Lint and the fast test tier. Linux runs on every push; macOS (where the extraction
+# sandbox exists) runs on demand, because private-repo macOS minutes cost 10x.
+# Actions are pinned by commit. No secrets are used, and the token is read-only.
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      macos:
+        description: "Also run on macOS (sandbox tests)"
+        type: boolean
+        default: false
+      onnx:
+        description: "Also run semantic search on Linux with the real model (1.2 GB download)"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: tests-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7  # v10.2.0
+        with:
+          version: "0.12.18"
+      # The bench group (torch, tiktoken) is for evaluation only; on Linux torch would
+      # pull non-permissive CUDA libraries. Two token-budget tests skip without tiktoken.
+      # The extras on Linux: semantic search through ONNX Runtime (issue #12).
+      - run: uv sync --locked --all-extras --no-group bench
+      - run: |  # the kernel's security modules and Landlock's version; the sandbox must be on (#10)
+          uname -r; cat /sys/kernel/security/lsm || true; echo
+          uv run --no-sync python -c "from lanternkb.extract import linux_sandbox as l, pool; print('landlock', l.abi()); assert pool.sandbox_available()"
+      - run: uv run --no-sync pytest -v -p no:cacheprovider tests/test_linux_sandbox.py
+      - run: uv run --no-sync ruff check .
+      - run: uv run --no-sync ruff format --check src tests packages
+      - run: uv run --no-sync pytest -q -rs -p no:cacheprovider
+      - run: uv run --no-sync python scripts/check_licences.py  # Linux's set of extras
+      - run: ./scripts/audit_deps.sh  # SBOM and known-vulnerability check (OSV)
+
+  linux-onnx:
+    if: github.event_name == 'workflow_dispatch' && inputs.onnx
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      LANTERNKB_MODELS_DIR: ${{ github.workspace }}/.models
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7  # v10.2.0
+        with:
+          version: "0.12.18"
+      - run: uv sync --locked --all-extras --no-group bench
+      # The pinned ONNX export, fetched by lanternkb's own pull (the only network code).
+      - run: uv run --no-sync lanternkb models pull embed:embeddinggemma-300m
+      - run: uv run --no-sync lanternkb models pull rerank:gte-modernbert
+      # Its vectors and scores must match references made on macOS (tests/fixtures/
+      # onnx_reference.json, rerank_reference.json).
+      - run: uv run --no-sync pytest -q -p no:cacheprovider -m slow tests/test_onnx_embedder.py tests/test_quick_search.py
+
+  macos:
+    if: github.event_name == 'workflow_dispatch' && inputs.macos
+    runs-on: macos-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7  # v10.2.0
+        with:
+          version: "0.12.18"
+      # The extras (OCR, audio, semantic search) are macOS-only; their licences are checked here.
+      - run: uv sync --locked --all-extras --no-group bench
+      - run: |  # says whether a sandboxed worker starts, before the tests
+          printf 'index = "%s/lk/index.db"\n' "$RUNNER_TEMP" > "$RUNNER_TEMP/lk.toml"
+          chmod 600 "$RUNNER_TEMP/lk.toml"
+          uv run --no-sync lanternkb --config "$RUNNER_TEMP/lk.toml" doctor || true
+      - run: uv run --no-sync pytest -q -p no:cacheprovider
+      - name: Sandbox denials (on failure)
+        if: failure()
+        run: /usr/bin/log show --last 10m --style compact --predicate 'eventMessage CONTAINS "deny"' | grep -E "python|Python" | tail -60
+      - run: uv run --no-sync python scripts/check_licences.py

lanternkb-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+.DS_Store
+.venv/
+__pycache__/
+*.egg-info/
+dist/
+
+# never commit indexes, configs or personal data
+*.db
+*.db-wal
+*.db-shm
+config.toml
+data/
+runs/
+.env*
+
+# third-party tool installs for the bake-off
+node_modules/

lanternkb-0.1.0/.gitleaks.toml

@@ -0,0 +1,14 @@
+# Repository secret-scan settings: gitleaks' default rules, plus documented allowlists.
+# The planted-secret evaluation's generators build FAKE secrets at run time from templates
+# (labelled synthetic; hard rule 2). Their templates look like secrets to a scanner.
+[extend]
+useDefault = true
+
+[[allowlists]]
+description = "Synthetic fake-secret generators (src/lanternkb/eval/secrets/planted_*.py)"
+paths = ['''src/lanternkb/eval/secrets/planted_[a-z0-9_]+\.py''']
+
+[[allowlists]]
+description = "Scanner rule tests with labelled, made-up example values (tests/test_secret_rules_*.py)"
+paths = ['''tests/test_secret_rules_[0-9_]+\.py''', '''tests/test_k8s_secret_data\.py''',
+         '''tests/test_extract_formats\.py''']

lanternkb-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

lanternkb-0.1.0/CLAUDE.md

@@ -0,0 +1,159 @@
+# lanternkb
+
+A local-only search tool over a person's own files, with a secret-safe ingestion layer, visibility tiers, an MCP server over stdio and a published evaluation. Public repo: `ankitb073/lanternkb`, **Apache-2.0**, released as 0.1.0 on 2026-10-12 ([decision 0028](docs/decisions/0028-apache-2-and-public-release.md)): shipped dependencies must stay permissive; contributions are accepted under Apache-2.0; posting an announcement or creating accounts still needs the owner's yes. It began as an idea in `ankitb073/hq` (`ideas/2026-09-local-knowledge-base.md`). The name `lanternkb` was chosen on 2026-09-29 and kept by the owner on 2026-10-03 after the name check (decision 0023).
+
+## Read first
+
+1. [docs/prior-art.md](docs/prior-art.md): what exists, and the gap. Then [docs/results/phase1-bakeoff.md](docs/results/phase1-bakeoff.md): what the bake-off measured.
+2. [docs/prfaq.md](docs/prfaq.md): the target and what would kill it.
+3. [docs/threat-model.md](docs/threat-model.md): what we protect and do not claim.
+4. [docs/hld.md](docs/hld.md) and [docs/lld/](docs/lld/): the design.
+5. [docs/efficiency.md](docs/efficiency.md): the budgets every part is measured against.
+6. [docs/roadmap.md](docs/roadmap.md): phases, and the release checklist.
+7. [docs/decisions/](docs/decisions/).
+
+Scope notes: email and browser history are **out of v1**. Every capability is an isolated, toggleable feature ([docs/lld/features-ui.md](docs/lld/features-ui.md)).
+
+## Current state (2026-10-12)
+
+**Phase 10 (public release), 2026-10-12, the owner's yes** ([decision 0028](docs/decisions/0028-apache-2-and-public-release.md)): Apache-2.0; the repo public with private vulnerability reporting; 0.1.0 of `lanternkb` and `lanternkb-ui` (the `ui` extra) through PyPI trusted publishing (`.github/workflows/release.yml`, on a `v*` tag). Install: `uv tool install --managed-python --python 3.12 "lanternkb[embed,ui]"` (a plain install on a fresh Mac picks Python 3.9 and fails). Added `lanternkb --version` and `tests/test_release.py` (one version everywhere, Apache-2.0 in every package). A clean-home install of the built wheels synced, searched, listed recent files and redacted a planted fake password. The announcement in `docs/launch/` is still the owner's to post.
+
+**Phases 1 to 8 are done; phase 9 (hardening) is done except what needs the user.** Start with the [evaluation report](docs/results/evaluation-report.md), which gathers every phase's results, failures included. Phase reports: [1](docs/results/phase1-bakeoff.md), [2](docs/results/phase2-safety.md), [3](docs/results/phase3-features.md), [4](docs/results/phase4-extractors.md), [5](docs/results/phase5-search.md), [6](docs/results/phase6-mcp.md), [7](docs/results/phase7-ui.md), [8](docs/results/phase8-plugins.md), [9 routing](docs/results/phase9-routing.md).
+
+- **Usable now** (README, "Try it"):
+  - `lanternkb serve --tier open|all`: the MCP server, checked live in Claude Code;
+  - `search`, `doctor`, `mcp-setup` (with suggested Claude Code deny rules), `install-schedule`, `--redact` for bug reports;
+  - `ui`: the settings page;
+  - plugins ([docs/plugins.md](docs/plugins.md)).
+- **Search:**
+  - Phase 5 blind test (330 queries): nDCG@10 **0.831**, against 0.760 for QMD and 0.777 for grooveseek with its reranker.
+  - Phase 9 blind test (300 fresh queries): `ROUTE9` 0.829 against 0.820 for the phase 5 routing, so `ROUTE9` is the default routing (`search.DEFAULT`).
+  - **Third blind test (2026-10-03, 300 queries; 29 repeat tuning queries, see the results' sensitivity table): reranking the top 20 files** (gte-reranker-modernbert-base, fused with the search's order at RRF 5; `lanternkb.rerank`, [decision 0022](docs/decisions/0022-rerank-top-files-by-default.md)) scored **0.885** against 0.865; +0.033 over the vector part, meaningful as pre-registered (+0.029 without the 29 repeats, just under the bar), so **#14 is closed** and reranking is the default on macOS (`models pull rerank:gte-modernbert`). 138 / 187 ms per search. **All three blind sets are spent.**
+  - **ONNX ([decision 0024](docs/decisions/0024-onnx-for-one-off-searches-and-linux-reranking.md), [results](docs/results/quick-search.md)):** terminal searches load the ONNX exports when pulled (`models pull ... --backend onnx`): 0.54 s instead of 2.88 s (1.90 s instead of 3.24 s with reranking), same dev scores (#33 closed). Query vectors may cross backends within EmbeddingGemma's 257-token window. Reranking through ONNX on a CPU takes 0.4 to 1 s, over budget, so on Linux it is **opt-in** (`features enable reranker.gte-modernbert`; #25 closed; checked in the `linux-onnx` CI job).
+  - Dev nDCG@10: 0.854 with reranking (0.836 without), in-process and through MCP (123 / 182 ms).
+- **Vectors (schema 8, [decision 0020](docs/decisions/0020-vectors-in-plain-tables-binary-shortlist.md)):** plain tables of float16 plus sign bits; an in-memory Hamming scan, then exact re-scoring of a shortlist (2% of visible, 4,000 to 20,000). 10 ms p50 at 100,000 chunks, 66 ms at 1,000,000; 363 MB per 100,000. sqlite-vec is loaded only to migrate old indexes.
+- **Safety (published number 71.9%):** planted secrets kept out, on held-out splits: 87.5% (phase 2), 82.7% (second), **71.9%** [68.5%, 75.1%] (third, raw text; 84.5% through the pipeline; [results](docs/results/secrets3.md)). Since the third, splits are scored **through the pipeline** (what reaches the index), the primary measure.
+  - Fourth ([results](docs/results/secrets4.md)): fresh seeds of three generators; the default kept 92.6% out; round 3 (decoding views, `scan.rules.ROUND3`) added one secret, so it stays **opt-in** with round 2 (`scan.rules.OPT_IN`, `[scanner] enabled_rules`).
+  - **Fifth (2026-10-03, [results](docs/results/secrets5.md)):** fresh seeds of all four generators, 2,450 secrets. **Round 4** (`scan.rules.ROUND4`: YAML fold, punctuation and escaped-JSON views, multi-line CSV cells joined only inside tokens, `allowlist-resume`, three narrow rules) kept 31 more out (91.3% against 90.0%, +1.27 points [+0.86, +1.76]), +0 on the family it was not built from, the same 4 of 401 clean files flagged: **on by default** ([decision 0025](docs/decisions/0025-round-4-on-by-default.md), `ENGINE_VERSION` 13). Review 21 (and a follow-up) found a P1 in it before the test (resume skipped default matches), fixed.
+  - **Sixth (2026-10-07, [results](docs/results/secrets6.md)):** fresh seeds of all four generators, 2,448 secrets. **Round 3** (decoding views, fixed after review 23 on today's engine) kept 43 more out (92.9% against 91.1%, +1.76 points [+1.27, +2.29]), all on its own family, no loss elsewhere: **on by default** ([decision 0026](docs/decisions/0026-round-3-on-by-default.md), `ENGINE_VERSION` 15). **Round 5** (`scan.rules.ROUND5`: Windows command lines, `setx`, a password after a few words, #36) added 2 and stays opt-in.
+  - **Seventh (2026-10-08, [results](docs/results/secrets7.md)):** fresh seeds of all four generators, 2,455 secrets. **Round 6** (rules for indexed formats, three views, Kubernetes Secrets in JSON, word-bounded stopwords; review 27 found 3 P1 and 11 P2 before the run, all fixed) kept 36 more out (95.7% against 94.3%, +1.47 points [+1.02, +1.96]), every family gaining or holding, the same 5 of 401 clean files flagged: **on by default** ([decision 0027](docs/decisions/0027-round-6-on-by-default.md), `ENGINE_VERSION` 17). The 4 more decoys removed are raw WPA keys the generator labels as decoys (disclosed before the run).
+  - Kubernetes Secrets with several values are redacted whole (`kubernetes-secret-data` pass, #34 closed).
+  - All seven secret splits and `clean2` to `clean6` are spent. The evaluation's leak check removes invisible separators from stored text first (review 23). Remaining misses (round 6, seventh split): punctuation after a secret, CSV without a keyword, a keyword far from its value, Markdown notes, CI and rotation logs, spreadsheet shared strings.
+  - The table pass (`engine.table_secrets`) never ends a table on a guess; reviews 15 to 19 repros are in `tests/test_secret_rules_1[5-9].py`.
+  - **Sandbox:** deny by default. Workers open no user files and cannot see other processes; the parent hands each file over and checks the kernel's path for it (#7). Workers are split by kind and by tier; open-tier media workers share no GPU state (decision 0021). A scanner-only process scans open-bound and plugin output again, with line breaks as stored (#15, review 11). Escaped helpers are found by their working folder (inside the worker's private one) and killed (#17, review 12).
+  - **Hangs fail closed:** plugin loading (30 s), tier rules (2 s each, 120 s per sync), query embeddings (30 s), embedding batches (900 s), reranking (10 s; 30 s on the CPU) (#16, #21).
+  - **Tiers:** bound only what the MCP server returns. The threat model says so, and `mcp-setup` prints Claude Code deny rules.
+- **Efficiency (phase 9):**
+  - 28 / 45 ms per search (35 / 49 ms through MCP);
+  - sync 55 s for 1,504 files with embeddings, about 1.5 s without;
+  - about 48 MB plus the model;
+  - with reranking (the default), 119 / 183 ms per search in-process, 123 / 182 ms through MCP (M5 Max); estimated over the 150 ms p50 budget at a million chunks;
+  - memory, counted as footprint (GPU included): embedder about 1.7 GB, reranker about 1.1 GB while loaded;
+  - every other budget met except: duplicates partly met; freshness partly met.
+- **Reviews:** twenty-seven independent reviews (phases 2 to 9, the schema 8 vector store, two of the phase 9 fixes, the Linux sandbox, reranking, five of the 2026-10-03 secret rules, round 3's decoding views, round 4 with a follow-up, the ONNX search path, on 2026-10-07 round 3 again, the new extractors and the product features, and on 2026-10-08 the browser search, round 6 and the day's product features, each re-checked) found and fixed 60 P1 and 142 P2 defects. Review 11's P1s led to schema 9 (`files.read_tier`: a file is shown as open only if read on an open worker) and [decision 0021](docs/decisions/0021-open-media-shares-no-gpu-state.md) (open-tier media use Tesseract and the CPU).
+  - **Phase 9 also had:** a threat-model audit; a fresh-clone quickstart test; CI on macOS (python.org's Python cannot load SQLite extensions, so `[tool.uv] python-preference = "only-managed"`).
+  - **Prompt injection (#19, closed):** Claude Opus 5.5 with only lanternkb's tools saw 12 kinds of planted instruction 129 times in 24 sessions and followed none ([results](docs/results/injection.md)).
+  - **Open issues:** #29 (small table-pass gaps and leftovers, P3), #36 (a password after a few words: caught only by opt-in round 5, P2), #39 (calendars and contacts not indexed until the scanner covers them, P3).
+  - **Linux (2026-10-02):** semantic search through ONNX Runtime (#12, dev 0.838) and a Landlock + seccomp sandbox for workers (#10, `extract/linux_sandbox.py`), both checked in Linux CI.
+- **Release checklist** ([roadmap](docs/roadmap.md)): every item that does not need the user passes, CI included.
+- **Added on 2026-10-07 (autopilot):**
+  - file types: OpenDocument (`extractor.opendocument`), rich text, web pages as readable text, subtitles, about 40 more code and markup types; extractor `version` in manifests re-reads a changed feature's files; touching inline texts are separated by U+2063, which the scan reads as a space and a default join view reads joined;
+  - search filters (`types`, `modified_after`, `modified_before` in MCP; `--type`, `--since`, `--before` in the CLI), pushed into the vector search;
+  - `lanternkb setup`, a guided first run;
+  - ONNX query vectors in the MCP server (#35 closed);
+  - `scripts/check_ui_layout.py` (settings page at real phone widths), `scripts/ci_local.sh`;
+  - search ideas tried on dev and not pursued: `bench/search-ideas-log.md`.
+- **Added on 2026-10-08 (autopilot, day 2):**
+  - **Search in the browser:** the settings page (`lanternkb ui`) has a search box.
+    - It searches every tier through the MCP server's own search; inside Claude Code, open files only.
+    - Queries go by POST, then a redirect to results kept under a random handle (Post/Redirect/Get), so they stay out of addresses, titles, history and the saved session.
+    - Matches are marked, and **Show in Finder** takes a result's id and path. Review 26 found no P1 and 2 P2s, fixed and re-checked.
+    - Design: [docs/lld/features-ui.md](docs/lld/features-ui.md), "Search on the settings page".
+  - **Terminal search** marks the query's words in bold. File text and names printed by `search`, `skipped` and `explain` lose control characters, so a file cannot send escape sequences to the terminal.
+  - **Result ids** carry a tag of the file's path (#37 closed).
+  - **Review 25 completed** (third look and verification):
+    - join-view windows keep whole lines (a P2 leak with line-anchored rules);
+    - a long run costs in proportion;
+    - windows never start inside a token;
+    - `ENGINE_VERSION` 16.
+  - **Round 6** (`scan.rules.ROUND6`; **on by default since the seventh test**, see Safety above):
+    - rules for `aws configure` transcripts, connection strings (ADO.NET, JDBC, libpq), exported and secret-named assignments, Wi-Fi keys, cloud-init, XML settings, LaTeX macros, Helm literals and FileZilla;
+    - LaTeX, prefix-wrap and emphasis views; Kubernetes Secrets in JSON (both kubectl copies);
+    - the vendored generic rule's short stopwords read as words in random-looking values.
+    - Review 27 found 2 P1s and 9 P2s before any test, all fixed.
+    - On spent splits it keeps 132 more of 6,114 out, with no new clean flags and 0 findings in 14,190 third-party files.
+    - Seventh held-out split: `test7`, seeds 9201–9204, `clean6` seed 8009; plan `bench/secrets7-plan.md`, used once.
+  - **Later on 2026-10-08 (after the owner's "keep going"):**
+    - **recent files:** a search for `*` (MCP, terminal, settings page) or `lanternkb recent` lists files newest first through the same tier and filter checks; MCP results carry `modified`;
+    - **sync progress:** a one-line counter on the terminal (TTY only) and the stage on the settings page; listeners hear counts, never paths;
+    - **a fresh-clone run** (independent agent, no P1) led to:
+      - suggested commands printed as the user would run them (`uv run`, `--config`; `lanternkb.hints`);
+      - unquoted multi-word queries;
+      - "No files skipped.";
+      - clearer deny-rule text;
+      - doc notes on the index folder and example-looking values;
+    - **#38 closed:** join-view findings widen on the text, so a glued token redacts only itself;
+    - **the join view again joins every separator in one reading** (a P2 its verification found: a secret split on both sides of punctuation);
+    - `ENGINE_VERSION` 19; development numbers unchanged on every spent split.
+    - **review 28** (recent files, hints, progress, UTF-16) found a P1:
+      - Text decoded as UTF-16 when the file mixes UTF-16 with other text (PowerShell `>>`) hid an ASCII secret as CJK characters.
+      - The worker now scans a NUL-stripped view of the raw bytes for text and code files with zero bytes or a UTF-16 BOM (`extract.worker.hidden_by_decoding`), and refuses the file (`mixed-encoding`) when that view holds a secret the decoded text does not show. This also closes the older BOM and UTF-8-head cases.
+      - Its P2 (recent files stalled on hidden files) is fixed with an mtime index, keyset paging and a 20,000-file cap.
+      - Its re-check found a P1 (the raw view kept CRLF) and two P2s (a duplicate hidden copy; keyset paging by path still sorted). All fixed: line breaks as scanned, a count comparison, paging by (time, row id). 1M files list in 0.1 ms.
+    - a count-only check of the Downloads folder (allowed 2026-10-03; [results](docs/results/real-files-1.md)): round 6 flags nothing new; calendars and contacts wait for scanner support (#39).
+- **Owner decisions (2026-10-07):** 24-hour autopilot: expand features, support and fixes on first principles, quality first, without pausing; Colima may be installed for Linux tests (done); short status notes may be added to existing decision records.
+- **Owner decisions (2026-10-03):** licence closed for now, to be thought through together; name `lanternkb` kept ([decision 0023](docs/decisions/0023-keep-the-name-lanternkb.md)); repo private for now; #19 may use Claude Code quota; real-file testing allowed on a few files (the Downloads folder), reporting aggregate numbers only.
+- **Needs the user:** where (and whether) to post the announcement; real-file use on their own folders.
+- **Tests:**
+  - `uv run pytest`: 1,104 fast, always with a temporary HOME and `CLAUDECODE` unset; slow tests are excluded by default (`addopts`).
+  - Slow: `LANTERNKB_MODELS_DIR=$PWD/data/cache/models uv run --all-extras pytest -m slow` (media, the adapters for QMD and grooveseek, the backtracking sweep, the real ONNX exports against references).
+  - Also: `scripts/check_licences.py` (with `--all-extras`), `scripts/audit_deps.sh` (SBOM and OSV), `scripts/scan_history.sh`, `scripts/check_secrets_regression.py` (`--split dev|test|test2|test3`) and `scripts/check_wheel.py` (the wheel ships no evaluation harness; `lanternkb-eval` comes from the dev-only `packages/lanternkb-dev`).
+  - CI: `.github/workflows/tests.yml` runs Linux on every push and macOS on demand (`gh workflow run tests.yml -f macos=true`). **GitHub Actions minutes ran out on 2026-10-07:** `scripts/ci_local.sh` runs the same checks on this machine (`mac`, `linux` in a Colima container from `scripts/ci/linux.Dockerfile`, `linux-onnx`); start the VM with `colima start` (it mounts only the repo and the job's scratch folder). Run it before every push.
+  - **Commit only after checking pytest's own exit code** (`rc=$?`), never through a `| tail` pipe.
+- **Evaluation:** `./bench/run-phase5.sh dev|test`. For a re-measure that leaves the published tables alone, call `lanternkb.eval.runner.run_system` with a scratch `runs_dir`. The evaluation freezes the phase 1 identifier splitter (`text.split_identifier_v1`). Models live in `data/cache/models`.
+- **Testing on this machine:**
+  - Every temporary config must set `index =`.
+  - Never launch a browser without `--use-mock-keychain --password-store=basic`.
+  - `tmutil` takes a second per call: the code writes the backup-exclusion attribute directly.
+- The user wants **quality first** (keep independent reviews), and **real-file testing will come later**, when the user chooses; never before.
+- **Decide routine questions yourself, and keep working without waiting** (user, 2026-10-01). Ask only when a decision changes the paradigm or a hard rule applies.
+- **Next:** any further secret-rule change needs an eighth held-out split (fresh seeds; the generators' contexts are all known, so new contexts would need a new generator); any further search change needs a fourth held-out query set (no dev idea has cleared the 0.01 bar yet). Then phase 10 materials, unpublished (a draft announcement is in `docs/launch/`).
+
+## What to do when the user says "go"
+
+Work through the phases in [docs/roadmap.md](docs/roadmap.md), **in order and one at a time**, committing after each. Stop after phase 1 and report, because its outcome (decision 0004) may change the rest of the plan.
+
+### Efficiency
+
+- Default to Sonnet-class effort. Use a stronger model only for design decisions and for reviewing safety-critical code.
+- Prefer scripts and tests over agents. Use at most **4 subagents** at once, only where the work is independent: for example, one arm of the bake-off per tool, and a separate agent that reproduces results from a fresh clone.
+- Keep sessions short and commit often so nothing depends on a long context.
+- Do not re-research what [docs/prior-art.md](docs/prior-art.md) already records; only update it.
+
+### Hard rules
+
+1. **Never read the user's real files.** Build and test on synthetic data. The user will run the real index. Do not open, list or summarise personal folders.
+2. **No personal data in this repo or its history:** no real names, addresses, email addresses (other than the commit identity), machine names, home paths or documents. Fixtures are synthetic; fake secrets are labelled and allow-listed.
+3. **No telemetry, no network calls in the core, no logging of document text, secrets or queries.**
+4. **$0.** No paid services or data. Anything paid needs the user's explicit OK and must unlock something truly prestigious (hq decision 0002).
+5. **The evaluation plan is fixed before running systems on the test split.** Report failures and nulls. Do not claim differences smaller than the spread.
+6. **Do not post anywhere or create accounts** without the user's explicit OK. The repo is public and releases go out by tag (decision 0028); a new version still needs the owner's OK.
+7. Log bugs as GitHub issues with a priority label (P1 / P2 / P3), written so a fresh session can pick them up.
+
+### Stop and ask before
+
+Indexing any real folder, granting or requesting macOS permissions, touching email or browser history, adding a shipped dependency that is not permissively licensed (no GPL, LGPL, AGPL, SSPL; see decision 0012), changing a decision record, or spending any money.
+
+### When done (or when stopping at a gate)
+
+- Update "Current state" here and the status line in `README.md`.
+- Update the status and a short summary in `~/projects/hq/ideas/2026-09-local-knowledge-base.md` (hq is for writing things up, so editing that file is fine), then commit and push both repos.
+- Give the user a short summary: what was done, what the measurements showed (including failures), and what needs a decision.
+
+## Conventions
+
+- Python with `uv`; `pyproject.toml` with a lockfile; `pytest` (fast tier: `uv run pytest`); Ruff. Type-annotate public interfaces.
+- Third-party tools under evaluation run only with a throwaway `HOME` (`lanternkb.eval.measure.sandbox_env`), never against the user's home.
+- Commit identity for this repo: `ankitb073`, `38665863+ankitb073@users.noreply.github.com`. Do not add attribution lines to commit messages or pull request descriptions.
+- One short decision file per significant decision in `docs/decisions/` (context, decision, why).
+- Numbers first, with dates and sources.

lanternkb-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,10 @@
+# Code of conduct
+
+lanternkb is a small project about privacy, so the first rule follows from that.
+
+- **Respect people's privacy.** Never post someone's files, paths, credentials or personal details, theirs or yours, in issues, discussions or pull requests. Reproduce problems with synthetic data.
+- **Be kind and specific.** Criticise code and claims, not people. Say what you measured and how.
+- **No harassment** of any kind, in any project space or in private messages about the project.
+- **Assume good faith,** and ask before concluding otherwise.
+
+The maintainer may edit or remove anything that breaks these rules, and may block people who keep breaking them. To report a problem privately, use the contact in [SECURITY.md](SECURITY.md).

lanternkb-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,41 @@
+# Contributing
+
+lanternkb is licensed under Apache-2.0. By submitting a contribution you license it under the same terms (Apache-2.0, section 5; [decision 0028](docs/decisions/0028-apache-2-and-public-release.md)). Plugins need no contribution: see [docs/plugins.md](docs/plugins.md).
+
+## Ground rules
+
+- Read [docs/threat-model.md](docs/threat-model.md) and the [decisions](docs/decisions/) first.
+- Extensions go through the [extension API](docs/lld/extension-api.md) and pass its contract tests.
+- Never add telemetry or network calls to the core. Never log document text, secrets or queries.
+- Fixtures must be synthetic. Never commit real documents, personal paths or real credentials.
+- Generate fake secrets at run time (see `tests/fakes.py`). A line that must hold a fake one carries a `# fake-secret: <why>` comment, which the repo's own scan (`scripts/scan_repo.py --history`) allow-lists. Committed results must hold repo-relative paths only.
+- Claims about quality or safety need a measurement that others can reproduce. Report failures too.
+- Every bug fix comes with a test that fails without it.
+
+## Working on lanternkb
+
+```sh
+uv sync                                   # development tools, the settings page, the example plugin
+uv run pytest                             # the fast tier (about a minute)
+uv run ruff check . && uv run ruff format --check src tests packages
+LANTERNKB_MODELS_DIR=<folder with the models> uv run --all-extras pytest -m slow   # OCR, audio
+uv run --all-extras python scripts/check_licences.py   # shipped dependencies are permissive
+./scripts/audit_deps.sh                   # SBOM and known vulnerabilities (OSV)
+./scripts/scan_history.sh                 # secrets and personal data in the whole history
+uv run python scripts/check_secrets_regression.py   # planted secrets still kept out (release)
+```
+
+Things that bite:
+- **Every config you write for a test or an experiment must set `index = "<temp folder>/index.db"`.** Without it lanternkb writes to `~/.local/share/lanternkb`. The test suite runs with a temporary `HOME`.
+- **Never index a real folder** while developing. Build synthetic files in a temporary folder.
+- Tests that check the macOS sandbox run only on macOS; CI runs Linux on every push and macOS on demand.
+- A headless browser, if you ever need one, takes `--use-mock-keychain --password-store=basic` and a throwaway profile; otherwise macOS shows a blocking keychain dialog. The settings-page tests use `http.client` instead.
+
+## Issues
+
+Every bug is a GitHub issue with a priority label:
+- **P1:** hurts users now (local-only text or a secret can leak, a restriction fails open).
+- **P2:** soon, or a partial bypass.
+- **P3:** polish.
+
+Write it so someone with no context can pick it up: where, how to reproduce, why it matters, and a proposed fix.

lanternkb-0.1.0/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

lanternkb-0.1.0/NOTICE

@@ -0,0 +1,5 @@
+lanternkb
+Copyright 2026 ankitb073
+
+This product is licensed under the Apache License, Version 2.0 (see LICENSE).
+Third-party code and data it ships or downloads are listed in THIRD_PARTY_NOTICES.md.