whatwasit 0.1.0__tar.gz

whatwasit-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,74 @@
+name: CI
+
+on:
+  push:
+    branches: [main, "feature/**"]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Cache embedding model
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/huggingface
+          key: minilm-onnx-${{ runner.os }}-${{ hashFiles('whatwasit/embedder.py', 'pyproject.toml') }}
+
+      - name: Install package and test deps
+        run: pip install -e ".[dev]"
+
+      - name: Prefetch ONNX embedding model
+        run: |
+          python - <<'PY'
+          from whatwasit.config import Config
+          from whatwasit.embedder import build_embedder
+          build_embedder(Config.default()).encode(["warmup"])
+          PY
+
+      - name: Run tests
+        env:
+          HF_HUB_OFFLINE: "1"
+          TRANSFORMERS_OFFLINE: "1"
+        run: pytest --cov=whatwasit --cov-report=term-missing
+
+  build:
+    name: Build wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install build backend
+        run: pip install build
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Smoke-test wheel install
+        run: |
+          pip install dist/*.whl
+          whatwasit --help
+          python -c "import whatwasit; print(whatwasit.__version__)"

whatwasit-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,61 @@
+name: Publish to PyPI
+
+# Publishes when you publish a GitHub Release (tag vX.Y.Z must match pyproject.toml).
+#
+# One-time setup:
+#   1. PyPI → Account settings → API tokens → "Add API token"
+#      - Scope: project "whatwasit" (or entire account for the first upload)
+#      - Copy the token (pypi-AgE...); you only see it once
+#   2. GitHub repo → Settings → Secrets and variables → Actions → New repository secret
+#      - Name:  PYPI_API_TOKEN
+#      - Value: paste the PyPI token
+#
+# Release flow:
+#   1. Bump version in pyproject.toml and whatwasit/__init__.py
+#   2. Commit and push main
+#   3. git tag v0.1.0 && git push origin v0.1.0
+#   4. GitHub → Releases → Draft new release → pick tag → Publish release
+
+on:
+  release:
+    types: [published]
+
+concurrency:
+  group: pypi-${{ github.event.release.tag_name }}
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    name: Build and publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Verify release tag matches package version
+        run: |
+          TAG="${{ github.event.release.tag_name }}"
+          VERSION="${TAG#v}"
+          PKG_VERSION=$(python - <<'PY'
+          import tomllib
+          print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])
+          PY
+          )
+          if [ "$VERSION" != "$PKG_VERSION" ]; then
+            echo "::error::Release tag ${TAG} (version ${VERSION}) does not match pyproject.toml (${PKG_VERSION})"
+            exit 1
+          fi
+
+      - name: Build package
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          password: ${{ secrets.PYPI_API_TOKEN }}

whatwasit-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+*.egg
+
+# Virtual envs
+.venv/
+venv/
+
+# hist local data / generated artifacts
+*.usearch
+*.db
+*.sqlite
+*.sqlite3
+.coverage
+htmlcov/
+
+# Editor / IDE
+.idea/
+.vscode/
+.cursor/
+
+# Worktrees (created during development, not part of the package)
+/.worktrees/
+
+# Local temp (verification artifacts)
+.verify_tmp/
+
+# Local-only notes, not for version control
+/FINAL_REPORT.md
+/PROJECT_CONTEXT.md

whatwasit-0.1.0/ARCHITECTURE.md

@@ -0,0 +1,187 @@
+# Architecture
+
+This document records the architecture decisions for `whatwasit` and the reasoning
+behind each one. It is the contract that the implementation must follow.
+
+## Goal recap
+
+A local-first, privacy-preserving CLI that searches shell history by
+meaning/intent. Group history into time/directory "sessions", embed each session
+locally with a small sentence-embedding model, store vectors in a local on-disk
+index, and do nearest-neighbour search at query time. No cloud, no API keys, no
+data leaves the machine.
+
+---
+
+## Decision 1: Language -- Python
+
+**Chosen: Python.** Rust was the alternative.
+
+The performance requirements are demanding on paper (index 10k commands in <30s;
+query <1s at 100k+ sessions) but the actual hot paths are all C/native code
+regardless of the host language:
+
+- Embedding runs in ONNX Runtime (C++).
+- Nearest-neighbour search runs in usearch (C++) or numpy (BLAS).
+
+The Python layer only does parsing, grouping, and SQLite I/O, none of which is a
+bottleneck at this scale. Crucially, **we embed per *session*, not per command**:
+10k commands collapse to a few hundred to low-thousand sessions, so we run far
+fewer (and batched) encodes than the raw command count suggests.
+
+Python also wins on iteration speed and ecosystem (`fastembed`, `usearch`,
+`sentence-transformers`) and is effectively required by Hard Requirement #7
+(`pip install whatwasit`). Rust would give a single static binary and marginally
+faster cold start, but offers no help meeting the latency targets, which are
+already met. Rust is noted in `FUTURE_IDEAS.md` as a future distribution path.
+
+## Decision 2: Embedding model & runtime -- all-MiniLM-L6-v2 on onnxruntime
+
+**Chosen: `sentence-transformers/all-MiniLM-L6-v2` (384-dim) run as an ONNX
+graph directly through `onnxruntime` + `tokenizers`, on CPU, fully offline.**
+
+The model itself is confirmed as the right choice for short, terse command-line
+text:
+
+- 384 dimensions, ~90MB ONNX on disk, single-digit-ms CPU encode when batched.
+- General-purpose semantic similarity model that handles short text well, which
+  matches our session documents (a handful of command lines plus the directory
+  name).
+
+**Runtime decision and the reason it changed.** The spec named
+`sentence-transformers`, which depends on PyTorch (200MB+ even CPU-only) and
+undercuts the lightweight promise. We first adopted `fastembed` (same model,
+ONNX, no PyTorch). However, **measured on the target CPU fastembed delivered
+only ~28 texts/sec**, which blew the budgets (indexing 10k commands took ~34s vs
+the 30s limit, and a query took ~2-3s vs the 1s limit). Profiling showed the
+same ONNX graph driven *directly* through `onnxruntime` -- with
+`graph_optimization_level=ORT_ENABLE_ALL` and `intra_op_num_threads = cpu_count`
+-- runs at **~600 texts/sec (~20x faster)** and produces **bit-for-bit identical
+vectors (cosine 1.0)**. So the embedder now drives onnxruntime itself:
+`tokenizers` for tokenization, the ONNX `model.onnx` for inference, mask-weighted
+mean pooling, then L2 normalization (exactly matching all-MiniLM-L6-v2). The
+ONNX weights + tokenizer are fetched once via `huggingface_hub` and cached.
+
+This is precisely the payoff of hiding the model behind the `Embedder` ABC
+(`encode(texts) -> np.ndarray`, rows L2-normalized): the runtime was swapped with
+zero changes to the indexer, search, or CLI. Swapping forward to an
+instruction-tuned asymmetric model (e5/bge) remains a one-class change. See
+`BENCHMARKS.md` for the before/after numbers.
+
+Known limitation: queries are natural language ("how did I fix the nginx issue")
+while documents are commands. MiniLM is roughly symmetric and works well enough
+for the MVP; asymmetric query/passage models are listed in `FUTURE_IDEAS.md`.
+
+## Decision 3: Vector storage -- usearch
+
+**Chosen: `usearch`**, behind the `VectorIndex` ABC. Candidates compared for the
+10k-100k+ vector scale:
+
+| Option | Verdict | Reasoning |
+|---|---|---|
+| `sqlite-vss` | Rejected | Effectively unmaintained (superseded by `sqlite-vec`); relies on loadable SQLite extensions with spotty/fragile wheels, and is weak on Python 3.13. Install friction directly threatens the clean `pip install` requirement. |
+| `hnswlib` | Viable, not chosen | Fast, mature HNSW, but forces us to hand-roll the key->metadata mapping and index persistence ourselves -- more glue, more ways to desync the index from the DB. |
+| `usearch` | **Chosen** | Single pip wheel across manylinux/macOS/Windows; HNSW *and* exact search; cosine metric; memory-mapped single-file save/load; integer keys map directly to our SQLite `session.id`; scales far past 100k. |
+
+**Scale sanity check:** at 100k vectors x 384 float32 (~150MB) even a brute-force
+numpy cosine scan is ~tens of milliseconds, so the <1s query target is met *even
+without* an ANN index. usearch is chosen not because ANN is strictly required at
+this scale, but for clean single-file persistence and headroom to grow. Because
+the store is behind `VectorIndex`, a numpy brute-force backend remains a trivial
+drop-in fallback.
+
+The vector file (`index.usearch`) holds vectors keyed by `session.id`; all
+session metadata and command text lives in SQLite. The two are linked solely by
+that integer key.
+
+## Decision 4: Session-grouping algorithm -- time gap + reconstructed cwd
+
+**Chosen: split on time gap OR working-directory change, with the working
+directory reconstructed from the command stream.**
+
+The spec's heuristic (commands within ~5 min AND in the same directory form one
+session) is sound, but it hits a hard reality: **plain `~/.zsh_history` and
+`~/.bash_history` do not record a working directory per command.** Only atuin
+stores cwd. So the directory half of the heuristic is not directly available for
+the primary data sources.
+
+Resolution:
+
+1. **Reconstruct cwd by replaying directory changes.** Walk commands in order,
+   maintaining a running cwd. Update it on `cd`, `pushd`, `popd`, and bare `cd`
+   (-> home). Resolve relative paths against the current cwd; when a target
+   cannot be resolved (e.g. `cd "$VAR"`, `cd $(...)`), keep the previous cwd and
+   mark uncertainty rather than guessing.
+2. **Use real cwd when available.** atuin records the actual cwd, so when reading
+   from atuin we use it directly instead of reconstructing.
+3. **Boundary rule.** Start a new session when the inter-command time gap exceeds
+   `session_window_seconds` (default 300) **or** (if `split_on_cwd_change`) the
+   cwd changes. Both knobs live in `Config`.
+4. **Timestamp-less sources.** bash without `HISTTIMEFORMAT` has no timestamps;
+   there we fall back to splitting on cwd change in file order only.
+
+This keeps the spec's intent (sessions = "what you were doing in one place around
+one time") while being honest about what the data actually contains.
+
+## Decision 5: Extensibility guardrails (applied, not over-built)
+
+- `Embedder` ABC wraps the model so it can be swapped without touching callers.
+- `VectorIndex` ABC wraps the store so usearch/hnswlib/numpy are interchangeable.
+- `schema_version` is written to the SQLite `meta` table from day one (currently
+  `1`).
+- All tunables (session window, cwd-split toggle, model name, dim, data paths,
+  top-k) are centralized in a single `Config` object, not scattered constants.
+- We deliberately do **not** build plugin systems, multi-backend config, or
+  speculative feature flags. Anything tempting goes to `FUTURE_IDEAS.md`.
+
+---
+
+## Data model
+
+SQLite (`whatwasit.db`) holds metadata; `index.usearch` holds vectors keyed by
+`session.id`.
+
+- `meta(key, value)` -- includes `schema_version`.
+- `sessions(id, start_ts, end_ts, cwd, command_count, doc_text)`.
+- `commands(id, session_id, seq, source, ts, duration, exit_code, cwd, raw_cmd)`.
+
+`Session.to_document()` is the single source of truth for the embedded text:
+the directory basename followed by the session's commands, in order.
+
+## Pipeline
+
+```mermaid
+flowchart LR
+  hfiles["~/.zsh_history / ~/.bash_history / atuin.db"] --> parser
+  parser["parsers -> Command[]"] --> grouper
+  grouper["sessions: time gap + cwd replay -> Session[]"] --> indexer
+  indexer["indexer: persist + embed + add to index"] --> store["SQLite + index.usearch"]
+  query["whatwasit 'natural language query'"] --> search
+  store --> search
+  search["search: embed query -> ANN -> hydrate"] --> output["rich output: ts, cwd, highlighted match + context"]
+```
+
+## Module boundaries
+
+- `whatwasit/config.py`, `whatwasit/models.py`, `whatwasit/db.py`, `whatwasit/interfaces.py` —
+  configuration, data model, persistence, and swappable subsystem interfaces.
+- `whatwasit/parsers/{base,zsh,bash,atuin}.py` — history file parsers.
+- `whatwasit/sessions.py` — session grouping (time gap + cwd reconstruction).
+- `whatwasit/embedder.py` — `OnnxEmbedder(Embedder)` (MiniLM via onnxruntime).
+- `whatwasit/index.py` — `UsearchIndex(VectorIndex)`.
+- `whatwasit/indexer.py` — parse → group → persist → embed → index orchestration.
+- `whatwasit/search.py` — query → embed → ANN → length normalization → gated hybrid
+  RRF reranking → hydrate → per-command match highlighting.
+- `whatwasit/cli.py`, `whatwasit/output.py` — CLI entry point and rich formatting.
+- `tests/synthetic.py` — synthetic multi-topic history generator (tests + bench).
+- `eval/` — offline search-quality evaluation harness (not shipped in the wheel).
+- `benchmarks/` — performance benchmark script (not shipped in the wheel).
+
+## Performance plan vs requirements
+
+- **Index 10k commands < 30s:** embedding is per-session (far fewer than 10k) and
+  batched through ONNX; SQLite writes are batched in a transaction.
+- **Query < 1s at 100k+ sessions:** one query encode (~ms) + usearch search
+  (sub-ms to low-ms); brute-force numpy would still be tens of ms.
+
+Real measured numbers are recorded in `BENCHMARKS.md`.

whatwasit-0.1.0/BENCHMARKS.md

@@ -0,0 +1,86 @@
+# Benchmarks
+
+Fully offline. Embedding model `all-MiniLM-L6-v2` (384-dim ONNX via
+`onnxruntime` + `tokenizers`), vector store `usearch`, SQLite metadata.
+
+Reproduce with:
+
+```bash
+HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 python benchmarks/run_bench.py --write-md
+```
+
+## Important note on the measurement machine
+
+The first numbers captured, in the "Measured" table below, came from an
+**11th Gen Intel Core i5-11320H that was thermally throttled to ~1.5-1.9 GHz**
+(nominal boost is 4.2 GHz) and under concurrent load at the time. Indexing time
+is dominated by CPU embedding throughput, so those figures were a pessimistic
+lower bound, not representative hardware. Both hard requirements are now
+**confirmed passing** on representative, un-throttled hardware — see
+"Confirmed on un-throttled hardware" below.
+
+## Measured (throttled i5-11320H @ ~1.5-1.9 GHz, under load)
+
+| Commands | Sessions | Index time | Query latency (avg) | Query latency (max) |
+|---:|---:|---:|---:|---:|
+| 1,000 | 109 | 11.33 s | 1.18 s | 1.48 s |
+| 10,000 | 1,087 | 88.41 s | 1.37 s | 1.85 s |
+| 100,000 | 10,870 | 464.10 s | 610.8 ms | 813.6 ms |
+
+(The 1k/10k query rows were measured while duplicate benchmark processes were
+still contending for the CPU; the 100k row ran with less contention, which is why
+its query latency is actually the *lowest* of the three at 611 ms avg.)
+
+**Contention is measurable.** Three benchmark passes happened to overlap on this
+machine, and the same 10k indexing workload landed at **33.7 s / 47.8 s / 88.4 s**
+depending on how many passes were fighting for the CPU at that moment. The
+best-observed 10k indexing (33.7 s) is already right at the 30 s target under
+merely partial contention, so an un-throttled, uncontended CPU clears it easily.
+
+## Clean component measurements (same machine, momentarily idle)
+
+These isolate the pieces that matter for the requirements and were captured when
+the CPU was briefly idle:
+
+| Component | Result |
+|---|---|
+| Embedder throughput (onnxruntime + tokenizers, batched) | **~608 texts/sec** |
+| Same model via `fastembed` (rejected runtime) | ~28 texts/sec (~40 max, even with `parallel=8`) |
+| Raw ONNX batched inference (256 x len-32) | ~163 seq/sec |
+| Nearest-neighbour search over **100,000** session vectors | **1.6 ms avg, 2.1 ms max** |
+
+Embedding is per **session**, not per command, so a session count is what drives
+indexing time. At the clean ~608 texts/sec, embedding the 1,087 sessions from
+10k commands is ~1.8 s; the rest of indexing (parse, group, SQLite writes,
+`usearch.add`) is well under a second.
+
+## Confirmed on un-throttled hardware
+
+Measured on an Intel i9-14900 (32 threads, boosts to 5.5 GHz), Ubuntu 22.04,
+idle system, single clean run:
+
+| Commands | Sessions | Index time | Query latency (avg) | Query latency (max) |
+|---:|---:|---:|---:|---:|
+| 1,000 | 109 | 0.80 s | 65.1 ms | 101.0 ms |
+| 10,000 | 1,087 | 7.19 s | 165.0 ms | 270.8 ms |
+| 100,000 | 10,870 | 92.89 s | 153.3 ms | 206.5 ms |
+
+Pure nearest-neighbour search over 100,000 session vectors: **0.7 ms avg, 0.9 ms max**.
+
+## Requirement checks
+
+- **HR#3 - index 10k commands < 30s: PASS, with 4x margin.** 7.19 s measured
+  (vs. 88.41 s throttled / 33.7-47.8 s partially-throttled). Indexing scales
+  with session count and clock speed as predicted from the clean component
+  measurements above, not anything specific to the throttled machine.
+- **HR#5 - query < 1s at 100k+ indexed sessions: PASS, with 5-6x margin.**
+  153.3 ms avg / 206.5 ms max at 100k sessions. Nearest-neighbour search itself
+  is negligible (0.7 ms); the ~150-200ms is dominated by the two small ONNX
+  forward passes per query (embedding the query text, then batch-embedding the
+  matched sessions' commands for highlighting) — a fixed per-query cost that is
+  independent of index size, which is exactly why query latency does not grow
+  from 10k to 100k sessions.
+
+Both hard requirements are now confirmed on representative hardware; the
+throttled numbers above remain as a record of why the embedder was rewritten
+from `fastembed` to direct `onnxruntime`, and as a worst-case lower bound.

whatwasit-0.1.0/FUTURE_IDEAS.md

@@ -0,0 +1,42 @@
+# Future Ideas (explicitly out of MVP scope)
+
+These are deliberately **not** built in v1. The MVP scope is fixed to the Hard
+Requirements. Good ideas are parked here instead of expanding scope.
+
+## Captured from the spec (Full Version)
+
+- **Background daemon / shell hooks for live indexing.** Embed and index new
+  commands as they are run, in the background, with zero perceptible latency.
+  This is also the clean way to capture the *real* working directory going
+  forward (instead of reconstructing it from `cd` replay).
+- **Fish shell support.** Reading `~/.local/share/fish/fish_history` (YAML-ish).
+- **Interactive mode.** fzf-style fuzzy re-filtering of the ranked results.
+- **Deeper atuin integration.** Beyond reading its DB: respect its richer
+  metadata (hostname, session id, exit status) for ranking.
+- **Git semantic search (`giths`).** Same architecture applied to commit
+  messages + first N lines of each diff.
+- **Distribution beyond pip.** Homebrew formula; a single static Rust binary for
+  the CLI front-end (the embedding/index core could stay Python or move to a
+  Rust core with ONNX + usearch bindings).
+
+## Engineering ideas surfaced during design
+
+- **Asymmetric query/passage embedding models** (e5, bge, gte) with
+  query/passage prefixes, to better match natural-language queries against
+  command documents. Trivial to try behind the existing `Embedder` ABC.
+- **Per-command vectors** in addition to per-session, for finer-grained
+  highlighting and "jump to the exact command" results.
+- **numpy brute-force `VectorIndex` backend** as a zero-native-dep fallback
+  (already trivially supported by the `VectorIndex` interface).
+- **Incremental indexing** (only embed sessions whose commands changed) once a
+  live daemon exists, using content hashes.
+- **Schema migrations** beyond the version stamp, once `schema_version > 1`.
+- **Secret redaction** on read (mask things that look like tokens/passwords in
+  the displayed output). Shell history is sensitive; worth doing carefully.
+- **Encryption / sync.** Explicitly out of scope (that is atuin's domain), noted
+  only for completeness.
+
+## Process note
+
+If a feature here starts to feel necessary for the MVP, stop and confirm scope
+before building it.

whatwasit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ThorOdinson246
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.