repolens-search 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repolens_search-0.11.0/.github/workflows/ci.yml +31 -0
- repolens_search-0.11.0/.gitignore +26 -0
- repolens_search-0.11.0/.repolens.toml +38 -0
- repolens_search-0.11.0/CHANGELOG.md +458 -0
- repolens_search-0.11.0/LICENSE +21 -0
- repolens_search-0.11.0/PKG-INFO +189 -0
- repolens_search-0.11.0/README.md +163 -0
- repolens_search-0.11.0/ROADMAP.md +25 -0
- repolens_search-0.11.0/benchmarks/acceptance.jsonl +18 -0
- repolens_search-0.11.0/docs/agent-rule-template.md +14 -0
- repolens_search-0.11.0/pyproject.toml +58 -0
- repolens_search-0.11.0/repolens/__init__.py +16 -0
- repolens_search-0.11.0/repolens/__main__.py +8 -0
- repolens_search-0.11.0/repolens/bench.py +291 -0
- repolens_search-0.11.0/repolens/chunk.py +171 -0
- repolens_search-0.11.0/repolens/cli.py +300 -0
- repolens_search-0.11.0/repolens/discover.py +66 -0
- repolens_search-0.11.0/repolens/find.py +321 -0
- repolens_search-0.11.0/repolens/frontmatter.py +105 -0
- repolens_search-0.11.0/repolens/index.py +509 -0
- repolens_search-0.11.0/repolens/lint.py +151 -0
- repolens_search-0.11.0/repolens/log.py +40 -0
- repolens_search-0.11.0/repolens/purpose.py +216 -0
- repolens_search-0.11.0/repolens/root.py +242 -0
- repolens_search-0.11.0/repolens/schema.py +118 -0
- repolens_search-0.11.0/repolens/semantic.py +588 -0
- repolens_search-0.11.0/repolens/templates.py +100 -0
- repolens_search-0.11.0/tests/test_repolens.py +1416 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ${{ matrix.os }}
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
os: [ubuntu-latest, macos-latest]
|
|
15
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python-version }}
|
|
22
|
+
- name: Install (package + dev tools)
|
|
23
|
+
run: |
|
|
24
|
+
python -m pip install --upgrade pip
|
|
25
|
+
pip install -e . ruff pytest mypy
|
|
26
|
+
- name: Lint (ruff)
|
|
27
|
+
run: ruff check .
|
|
28
|
+
- name: Type-check (mypy)
|
|
29
|
+
run: mypy repolens/
|
|
30
|
+
- name: Test (pytest)
|
|
31
|
+
run: pytest -q
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# repolens index cache — regenerated from the filesystem, never committed
|
|
2
|
+
.repolens/
|
|
3
|
+
|
|
4
|
+
# Python
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
.venv/
|
|
8
|
+
venv/
|
|
9
|
+
dist/
|
|
10
|
+
build/
|
|
11
|
+
*.egg-info/
|
|
12
|
+
.pytest_cache/
|
|
13
|
+
.ruff_cache/
|
|
14
|
+
|
|
15
|
+
# Local design/working docs — next-phase spec, kept out of the public repo
|
|
16
|
+
HANDOFF.md
|
|
17
|
+
HANDOFF-*.md
|
|
18
|
+
VISION.md
|
|
19
|
+
.venv-*/
|
|
20
|
+
|
|
21
|
+
# OS
|
|
22
|
+
.DS_Store
|
|
23
|
+
|
|
24
|
+
# Editors / IDEs
|
|
25
|
+
.idea/
|
|
26
|
+
.vscode/
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# repolens config. The presence of this file marks the repo root.
|
|
2
|
+
# Docs: https://github.com/hawkesj12/repolens
|
|
3
|
+
|
|
4
|
+
[repolens]
|
|
5
|
+
# index_path = ".repolens/index.db" # default; a gitignored, disposable cache
|
|
6
|
+
# skip_dirs = ["build", "vendor"] # ADDED to sensible defaults (.git, node_modules, ...)
|
|
7
|
+
# skip_files = ["CHANGELOG.md"]
|
|
8
|
+
# code_exts = [".py", ".ts", ".go"] # override the default code-file extensions
|
|
9
|
+
# include_gitignored = true # index gitignored file CONTENT too (default: false —
|
|
10
|
+
# .gitignore is respected, so secrets/.env stay out).
|
|
11
|
+
# Turn on for a personal/knowledge repo whose notes are
|
|
12
|
+
# gitignored and you WANT searchable. (DB schema is
|
|
13
|
+
# indexed regardless — names only, via [integrations.sqlite].)
|
|
14
|
+
|
|
15
|
+
# Typed records: folder -> type. `recursive` classifies subfolders too.
|
|
16
|
+
# `exclude` globs drop artifacts. `require` = regex patterns a conforming doc must
|
|
17
|
+
# contain (a warn if missing). An explicit frontmatter `type:` overrides the folder.
|
|
18
|
+
#
|
|
19
|
+
# [types.doc]
|
|
20
|
+
# folder = "docs"
|
|
21
|
+
# recursive = true
|
|
22
|
+
# exclude = ["*draft*"]
|
|
23
|
+
# require = ["^# "] # e.g. must have an H1
|
|
24
|
+
|
|
25
|
+
# Optional: also index SQLite DB table/column names (schema only, read-only).
|
|
26
|
+
# `repolens init` AUTO-DISCOVERS databases and fills this in; edit by hand too.
|
|
27
|
+
# [integrations.sqlite]
|
|
28
|
+
# paths = ["data/app.db", "data/other.db"] # one or many (legacy `path = "..."` also works)
|
|
29
|
+
|
|
30
|
+
# Toolchain `repolens env` reports as PRESENT (with versions). `repolens init`
|
|
31
|
+
# auto-seeds this from your manifests (pyproject -> python, package.json -> node,
|
|
32
|
+
# ...). Edit freely — absence is the default, so only list what matters.
|
|
33
|
+
# [env]
|
|
34
|
+
# tools = ["git", "python", "node"]
|
|
35
|
+
|
|
36
|
+
# auto-seeded from this repo's manifests by `repolens init`
|
|
37
|
+
[env]
|
|
38
|
+
tools = ["git", "python"]
|
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/), and this project adheres to
|
|
5
|
+
[Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- **`repolens index --threads N`** — override `[semantic].threads` (the fastembed CPU-core
|
|
12
|
+
cap) for one build; `--threads 0` uses all cores for a fast one-off rebuild.
|
|
13
|
+
- **Back-compat read of a pre-0.11 `.repometa.toml`.** When `.repolens.toml` is absent,
|
|
14
|
+
repolens reads a legacy `.repometa.toml` with a one-time deprecation warning, so an
|
|
15
|
+
un-migrated repo keeps its config instead of silently falling back to defaults.
|
|
16
|
+
- **Private, opt-in event logging (`[log].enabled`).** When on, repolens appends one
|
|
17
|
+
JSON line per `find` (query, mode, hits + scores, timing) and per embed (file, chunk
|
|
18
|
+
count, model, timing) to `.repolens/events.jsonl` — inside the gitignored cache dir,
|
|
19
|
+
so it stays local and is never committed. Off by default; writes never raise (a
|
|
20
|
+
logging failure can't break a search). The find log accumulates the real queries run
|
|
21
|
+
against a repo — future material for growing the benchmark from actual usage. New
|
|
22
|
+
`repolens/log.py`; hooked in `cmd_find` and `semantic.embed_doc`.
|
|
23
|
+
- **`find` shows the matching passage with each hit.** A result now carries the
|
|
24
|
+
passage that actually matched, not just the file path — a semantic hit shows its
|
|
25
|
+
best-matching chunk, a lexical hit shows the FTS5 excerpt around the terms, trimmed
|
|
26
|
+
to one line. The winning chunk was already found and stored; the per-doc rollup was
|
|
27
|
+
discarding it. `cmd_find` prints it under each hit; `--json` gains a `snippet` field.
|
|
28
|
+
- **`repolens bench` + a committed gold set (`benchmarks/acceptance.jsonl`).** The
|
|
29
|
+
reproducible answer to "does ranking, and then the semantic half, actually help?":
|
|
30
|
+
18 query→gold-doc pairs across exact / conceptual / paraphrase classes, scored as
|
|
31
|
+
recall@k + MRR in THREE modes against the same corpus — a literal **grep** baseline,
|
|
32
|
+
**lexical** (BM25), and **hybrid** (BM25 + semantic), so the table reads as the
|
|
33
|
+
progression grep → BM25 → hybrid. Measured on this repo's own corpus (bge-base, k=8):
|
|
34
|
+
overall recall@8 grep 78% / lexical 50% / hybrid 100%; MRR 0.39 / 0.34 / 0.67. Grep
|
|
35
|
+
beats the lexical arm (it reads full file bodies, which the code index reduces to a
|
|
36
|
+
purpose-line + docstring), but hybrid beats grep decisively.
|
|
37
|
+
- **Code docstrings are indexed and embedded.** A code file used to be searchable by
|
|
38
|
+
ONE line (the extracted purpose-line); a three-way bench against ripgrep showed
|
|
39
|
+
grep beat hybrid on conceptual queries (MRR 0.518 vs 0.333) solely because grep
|
|
40
|
+
reads the docstrings the index threw away. `purpose.extract_doc` now keeps the
|
|
41
|
+
full module docstring / leading comment block (capped at 1500 chars) as the code
|
|
42
|
+
file's BM25 body + embedded text; the one-line purpose stays the display title.
|
|
43
|
+
After the change hybrid leads grep overall (MRR 0.640 vs 0.548, recall@8 100% vs
|
|
44
|
+
96%) and on the paraphrase (0.442 vs 0.233) and exact (1.000 vs 0.875) classes;
|
|
45
|
+
grep keeps a slim conceptual-MRR edge (0.536 vs 0.479), cut from a 0.185 gap to
|
|
46
|
+
0.057. Existing indexes need one `repolens index --rebuild` to pick this up (a
|
|
47
|
+
content hash can't detect an extraction-rule change).
|
|
48
|
+
- **Code purpose-lines are embedded** (one short chunk per code file). Dense retrieval
|
|
49
|
+
previously covered markdown only, so on a code repo the semantic half could only
|
|
50
|
+
surface prose docs and RRF demoted correct code hits (e.g. "nearest neighbor vector
|
|
51
|
+
lookup" ranked `semantic.py` #1 lexical but #4 hybrid; it now ranks #1 hybrid). A
|
|
52
|
+
code file with no extractable purpose-line keeps its BM25 path/filename signal and
|
|
53
|
+
simply gets no vector.
|
|
54
|
+
|
|
55
|
+
### Fixed
|
|
56
|
+
|
|
57
|
+
- **`find` no longer hangs behind a proxy that black-holes HuggingFace.** A network that
|
|
58
|
+
accepts the connection then drops it made the first model load hang with no timeout, and
|
|
59
|
+
the per-process failure flag re-hung every subsequent `find`. Now: a HuggingFace
|
|
60
|
+
read-timeout fails a stalled connection fast (without aborting a slow-but-flowing
|
|
61
|
+
download), a wall-clock backstop (env `REPOLENS_MODEL_LOAD_TIMEOUT`) catches any other
|
|
62
|
+
stall, and the failure is **persisted to a TTL sentinel file** so sibling processes
|
|
63
|
+
degrade to lexical immediately instead of re-hanging — self-healing after the TTL.
|
|
64
|
+
- **`repolens index` degrades on a locked index instead of crashing.** Under write-lock
|
|
65
|
+
contention it raised an uncaught `database is locked` traceback (while `find` already
|
|
66
|
+
degraded); it now prints "index busy" and exits non-zero.
|
|
67
|
+
- **No thundering-herd re-embed under concurrency.** One changed file made every concurrent
|
|
68
|
+
`find` redundantly re-embed it off a stale pre-lock snapshot; the write phase now
|
|
69
|
+
re-reads the current stored hash under the lock and skips a file a peer already committed.
|
|
70
|
+
- **Enabling semantic on a lexically-built index now backfills embeddings.** An index
|
|
71
|
+
built lexical-first (or after a model/dims change) silently kept running lexical while
|
|
72
|
+
reporting hybrid, because incremental indexing only touches _changed_ files. An
|
|
73
|
+
embedding signature (`model:dims`) is now stored in `meta`; on a mismatch (including an
|
|
74
|
+
index with no vectors, or one whose embeds silently failed), `build_incremental` forces
|
|
75
|
+
a full backfill so hybrid actually runs.
|
|
76
|
+
- **A model-load failure degrades once instead of a per-doc retry storm.** `_model()` only
|
|
77
|
+
cached on success, so an offline/cold-cache box re-attempted the load for _every_ doc —
|
|
78
|
+
an effective hang. The failure is now memoized (`available()` flips to lexical), announced
|
|
79
|
+
once, and fastembed's loguru noise is quieted so a graceful degrade doesn't read as a crash.
|
|
80
|
+
- **A no-op incremental refresh no longer takes the write lock.** It used to `BEGIN
|
|
81
|
+
IMMEDIATE` + commit even when nothing changed, serializing concurrent `find`-refreshers;
|
|
82
|
+
it now detects "nothing changed" under a read connection and returns without a write.
|
|
83
|
+
- **repolens no longer indexes its own `.repolens.toml`.** The config file is tooling,
|
|
84
|
+
not corpus, but it was being indexed and showed up as noise in results (the very first
|
|
85
|
+
`find` a new user runs). Added `CONFIG_NAME` to the default `skip_files`.
|
|
86
|
+
- **`find` no longer crashes when the embedding endpoint is down at query time.** The
|
|
87
|
+
pre-flight availability check is config-only, so a dead bring-your-own http endpoint
|
|
88
|
+
used to surface as a raw `EmbeddingError` traceback from the dense KNN; `find` now
|
|
89
|
+
degrades to lexical-only for that search and says so once on stderr.
|
|
90
|
+
- **Chunking is fence-aware.** The heading splitter treated `#` comment lines inside
|
|
91
|
+
` ``` `/`~~~` code fences as Markdown headings and cut fenced snippets apart
|
|
92
|
+
mid-block, degrading their embeddings (this repo's own CHANGELOG chunked into 14%
|
|
93
|
+
fewer, cleaner pieces after the fix). Heading detection is now suspended inside a
|
|
94
|
+
fence.
|
|
95
|
+
|
|
96
|
+
### Changed
|
|
97
|
+
|
|
98
|
+
- **Fairer grep baseline + reported uncertainty in `bench`.** The grep arm now ranks by
|
|
99
|
+
**distinct query terms matched** (not sum-of-raw-counts, which understated a reasonable
|
|
100
|
+
grep by ~30%), and `bench` prints a deterministic bootstrap **95% CI** on the MRR deltas.
|
|
101
|
+
The honest result: hybrid clearly beats BM25 (ΔMRR +0.33, CI [+0.17, +0.50], excludes
|
|
102
|
+
zero — this is what justifies default-on), but against a fair grep its MRR edge is within
|
|
103
|
+
noise (ΔMRR +0.16, CI [−0.01, +0.33]); hybrid's real win over grep is recall (100% vs
|
|
104
|
+
94%) and the meaning-based queries grep can't reach.
|
|
105
|
+
- **Benchmark framing softened + Ollama recommended for heavy use.** The README no longer
|
|
106
|
+
leads with "100% recall@8" (which flatters a 22-file corpus); it leads with MRR + the
|
|
107
|
+
k-stable margin and frames the gold set as a directional signal, not a claim. Added
|
|
108
|
+
guidance: for query-heavy agent use, point `[semantic].provider = "http"` at a resident
|
|
109
|
+
Ollama so the model isn't reloaded per `find` (most of a hybrid query's latency).
|
|
110
|
+
- **mypy is now part of CI** (the repo's own type gate; previously ruff + pytest only).
|
|
111
|
+
- **Semantic search is now a default dependency — `pip install repolens` gives hybrid
|
|
112
|
+
out of the box.** `fastembed` + `sqlite-vec` + `numpy` moved from the opt-in
|
|
113
|
+
`[semantic]` extra into the core dependencies, because a new user who missed the extra
|
|
114
|
+
silently got the weaker lexical-only experience. Lexical (BM25) remains — as the
|
|
115
|
+
degrade-fallback when the model can't load and as the benchmark's control — and
|
|
116
|
+
`[semantic].enabled = false` still opts out of the model. The `[semantic]` extra is
|
|
117
|
+
kept as an empty alias so existing install commands still resolve. The core is no
|
|
118
|
+
longer stdlib-only; first run downloads a ~200MB model once (cached under
|
|
119
|
+
`~/.cache/repolens`).
|
|
120
|
+
- **Renamed the on-disk footprint `.repometa` → `.repolens`.** The config file is now
|
|
121
|
+
`.repolens.toml` and the disposable index cache is `.repolens/` (was `.repometa.toml`
|
|
122
|
+
/ `.repometa/`) — one consistent name matching the tool. The cache is gitignored and
|
|
123
|
+
regenerates, so no data migration is needed; an existing repo just needs its config
|
|
124
|
+
renamed (`git mv .repometa.toml .repolens.toml`) and the stale `.repometa/` removed on
|
|
125
|
+
the next `repolens index`.
|
|
126
|
+
|
|
127
|
+
### Removed
|
|
128
|
+
|
|
129
|
+
- **The agent-orientation machinery — `repolens` is now search + lint, one thing done
|
|
130
|
+
well.** Removed the generated-rule / repo-map subsystem and everything that fed it:
|
|
131
|
+
the `refresh`, `map`, `rule`, `hook`, `enrich`, `digest`, and `env` subcommands; the
|
|
132
|
+
`[map]` and `[enrich]` config blocks; and the SessionStart/SessionEnd hook install
|
|
133
|
+
path. The map tried to keep a frozen "what lives here" blob fresh via a change-key
|
|
134
|
+
gate, which drifted (folder-granular key vs file-granular content) and — with a
|
|
135
|
+
`[map].command` set — shelled out to a model from committed config (a code-execution
|
|
136
|
+
surface). The retrieval half already does this better on demand: `repolens find`
|
|
137
|
+
answers "where does X live" per query, always current, no stored artifact to rot.
|
|
138
|
+
`enrich` (model-written metadata) went with it — the semantic tier makes hand-filled
|
|
139
|
+
descriptions unnecessary for recall. `init` no longer installs any agent config; it
|
|
140
|
+
scaffolds the config, index, and pre-commit lint hook. The removed code is preserved
|
|
141
|
+
on the `archive/map-machinery` branch. Distribution is a plain CLI (pipx/PyPI) — not
|
|
142
|
+
a Claude Code plugin. The bench gold set dropped its 6 queries that targeted the
|
|
143
|
+
removed modules (24 → 18), so its numbers reflect only the surface that still ships.
|
|
144
|
+
|
|
145
|
+
## [0.9.0] — 2026-07-15
|
|
146
|
+
|
|
147
|
+
The semantic release: `find` becomes hybrid, and the invisible session digest
|
|
148
|
+
becomes one visible, self-maintaining rule. Strictly additive to the existing
|
|
149
|
+
SQLite/FTS5 index — no storage migration, and the core stays stdlib-only.
|
|
150
|
+
|
|
151
|
+
### Added
|
|
152
|
+
|
|
153
|
+
- **Hybrid `find` (BM25 + semantic, RRF).** Dense retrieval fuses with the existing
|
|
154
|
+
BM25 ranking via Reciprocal Rank Fusion (k=60): BM25 carries exact-term/identifier
|
|
155
|
+
precision, embeddings carry paraphrase/meaning recall, and RRF combines the two
|
|
156
|
+
per-document ranked lists with no score normalization. `--lexical` forces BM25-only.
|
|
157
|
+
(No committed benchmark yet: the hybrid's paraphrase-recall benefit is a directional
|
|
158
|
+
signal from early hand-checks on a docs-heavy corpus, not a measured result, and RRF
|
|
159
|
+
can occasionally re-rank a strong BM25 hit rather than only adding to it — a
|
|
160
|
+
reproducible query→gold benchmark with a runnable scorer is the next step.)
|
|
161
|
+
- **Semantic tier as an opt-in extra (`pip install 'repolens[semantic]'`).** Embeddings
|
|
162
|
+
via `fastembed` (ONNX, CPU, no service — no Ollama), default model
|
|
163
|
+
`BAAI/bge-base-en-v1.5` (768-dim, built for short-passage retrieval). Vectors store
|
|
164
|
+
in the same index through `sqlite-vec` (fast `vec0` KNN) with a **numpy brute-force
|
|
165
|
+
cosine fallback** when a Python `sqlite3` build can't load the extension — so semantic
|
|
166
|
+
search works everywhere; the active path is announced.
|
|
167
|
+
- **Section-bounded chunking.** Docs split on Markdown heading boundaries — a chunk
|
|
168
|
+
never crosses a heading; a section within ~512 tokens is one chunk, a longer one is
|
|
169
|
+
packed into ~512-token pieces within the section. Chunks embed and roll up to their
|
|
170
|
+
best (min-distance) parent document, so per-doc BM25 and per-chunk vectors are
|
|
171
|
+
fusable. Only changed files re-embed (keyed off the existing content hash); a deleted
|
|
172
|
+
doc's chunks/vectors cascade away.
|
|
173
|
+
- **CPU throttle + bring-your-own embedder.** `[semantic].threads` caps fastembed's CPU
|
|
174
|
+
so a big first build stays gentle. `[semantic].provider = "http"` routes embedding to
|
|
175
|
+
any OpenAI-compatible `/v1/embeddings` endpoint (local Ollama/LM Studio or a metered
|
|
176
|
+
API) via stdlib `urllib` — key from an env var, never stored in config. Alternate
|
|
177
|
+
models (`nomic-embed-text-v1.5`, `-Q`) get their required task prefixes automatically.
|
|
178
|
+
- **The self-maintaining rule (`.claude/rules/repolens.md`).** A static header (what /
|
|
179
|
+
when / who / why / how) plus generated, delimited **Environment** (toolchain) and
|
|
180
|
+
**Map** (folder tree + DB schema) sections — a visible, openable file that replaces
|
|
181
|
+
the old invisible SessionStart digest.
|
|
182
|
+
- **`repolens refresh` — the early-cutoff change-detector.** The SessionStart hook now
|
|
183
|
+
runs this: it compares a `hash(folder-set + DB schema + toolchain)` change-key to the
|
|
184
|
+
one stored in the rule and regenerates only the Map/Environment blocks on a real
|
|
185
|
+
change (a ~no-op otherwise, atomic write). The static header is never touched.
|
|
186
|
+
- **`init` warm-build.** `repolens init` now builds the index immediately (including
|
|
187
|
+
embeddings when the extra is installed), so the first session isn't cold.
|
|
188
|
+
|
|
189
|
+
### Changed
|
|
190
|
+
|
|
191
|
+
- **SessionStart hook command is now `repolens refresh`** (was `repolens digest &&
|
|
192
|
+
repolens env`); the rule carries the map + toolchain, so the map is visible instead
|
|
193
|
+
of injected. `digest` and `env` remain as standalone probes.
|
|
194
|
+
- repolens no longer indexes its own generated rule file (`.claude/rules/repolens.md`).
|
|
195
|
+
- `SCHEMA_VERSION` → 1.3 (added the `[semantic]` config block + chunks/vectors tables).
|
|
196
|
+
|
|
197
|
+
## [0.8.0] — 2026-07-14
|
|
198
|
+
|
|
199
|
+
Hardening pass from an independent three-lens review, ahead of a PyPI release.
|
|
200
|
+
|
|
201
|
+
### Fixed
|
|
202
|
+
|
|
203
|
+
- **Root resolution no longer leaks the wrong repo (blocker).** `find_root()` was
|
|
204
|
+
anchored partly to the install location (`__file__`), so an editable / venv-in-repo
|
|
205
|
+
install could resolve, index, search — and, via `enrich`, **write to** — a different
|
|
206
|
+
repo than your working directory. Resolution is now anchored only to your cwd (or an
|
|
207
|
+
explicit start path). Safe to `pip install` however you like, not just via `pipx`.
|
|
208
|
+
|
|
209
|
+
### Security
|
|
210
|
+
|
|
211
|
+
- **`.gitignore` boundary is honest outside a git repo.** Ignore rules are enforced via
|
|
212
|
+
`git`, so in a **non-git** directory a `.gitignore` was silently not honored. repolens
|
|
213
|
+
now prints a clear stderr warning when it indexes a non-git directory that has a
|
|
214
|
+
`.gitignore`, instead of exposing ignored files without notice.
|
|
215
|
+
- **Symlinks are no longer followed out of the repo.** A file symlink (e.g. to
|
|
216
|
+
`/etc/passwd` or `~/.ssh/id_rsa`) was read and indexed — a real leak for a tool that
|
|
217
|
+
feeds an agent's context. Symlinked files are now skipped during indexing.
|
|
218
|
+
|
|
219
|
+
### Added
|
|
220
|
+
|
|
221
|
+
- **`max_file_bytes` (default 5 MB).** Files larger than the cap are skipped at index
|
|
222
|
+
time, guarding against a stray huge file (a generated dump, a vendored blob) bloating
|
|
223
|
+
the disposable index and reading unbounded bytes into memory. Config-overridable.
|
|
224
|
+
|
|
225
|
+
### Changed
|
|
226
|
+
|
|
227
|
+
- **Stemmed search.** The FTS5 index now uses the `porter unicode61` tokenizer, so
|
|
228
|
+
`find "ranking"` matches a file whose text says `ranked`. (Identifiers aren't split —
|
|
229
|
+
search `parse`, not `parseFrontmatter`.) Run `repolens index --rebuild` once to pick up
|
|
230
|
+
stemming on an existing index.
|
|
231
|
+
- **Any-term broaden on a zero all-term match.** A multi-word `find` is implicit-AND
|
|
232
|
+
(every term must appear in one file); when that yields nothing, repolens now retries as
|
|
233
|
+
any-term (OR) and says so on stderr, instead of silently returning no hits.
|
|
234
|
+
|
|
235
|
+
## [0.7.2] — 2026-07-14
|
|
236
|
+
|
|
237
|
+
### Fixed
|
|
238
|
+
|
|
239
|
+
- **`enrich --force` no longer stacks a second docstring on code files.** Code
|
|
240
|
+
purpose lines are fill-only even under `--force` — an existing docstring is
|
|
241
|
+
authoritative, so `--force` regenerates only DOC frontmatter (which enrich owns)
|
|
242
|
+
and leaves code docstrings alone, instead of prepending a duplicate.
|
|
243
|
+
|
|
244
|
+
## [0.7.1] — 2026-07-14
|
|
245
|
+
|
|
246
|
+
### Added
|
|
247
|
+
|
|
248
|
+
- **`enrich` command provider** — `[enrich].command` runs any CLI that reads the
|
|
249
|
+
prompt on stdin and prints the answer (takes precedence over the HTTP endpoint).
|
|
250
|
+
Set `command = "claude -p --model haiku"` to enrich on your **Claude subscription**
|
|
251
|
+
— no API key, and the compute runs off your machine, not a local model pegging
|
|
252
|
+
your CPU. Any other prompt→text CLI works too. Still stdlib-only.
|
|
253
|
+
|
|
254
|
+
### Fixed
|
|
255
|
+
|
|
256
|
+
- **`enrich --force` preserves a doc's other frontmatter keys** — it regenerates the
|
|
257
|
+
managed fields (description/domain/tags) by key-merge now, instead of rewriting the
|
|
258
|
+
whole block, so unrelated keys survive a force pass.
|
|
259
|
+
- Rule doc: header calls out "lexical/BM25"; the `<!-- repolens:rule -->` marker moved
|
|
260
|
+
to the bottom so the file opens with its heading.
|
|
261
|
+
|
|
262
|
+
## [0.7.0] — 2026-07-14
|
|
263
|
+
|
|
264
|
+
### Added
|
|
265
|
+
|
|
266
|
+
- **`repolens rule` — teach the agent to _use_ repolens.** A search tool the agent
|
|
267
|
+
doesn't know to reach for is dead weight; repolens shipped the capability but never
|
|
268
|
+
the instruction. `rule` writes a short routing rule ("concept → `repolens find`;
|
|
269
|
+
exact string → `rg`") where an agent actually reads it: `.claude/rules/repolens.md`
|
|
270
|
+
(auto-loads every session in Claude Code) or `AGENTS.md` at the repo root (the
|
|
271
|
+
cross-agent convention). Idempotent + **non-destructive** — skips if already
|
|
272
|
+
present, appends to an existing `AGENTS.md`, never clobbers. `repolens init`
|
|
273
|
+
installs it by default in a Claude Code repo (`--no-rule` opts out); `rule --check`
|
|
274
|
+
dry-runs. This is the missing instruction half of the tool.
|
|
275
|
+
|
|
276
|
+
## [0.6.1] — 2026-07-14
|
|
277
|
+
|
|
278
|
+
### Fixed
|
|
279
|
+
|
|
280
|
+
- **`enrich` no longer writes a bogus `domain` on repo-root files** — a file with
|
|
281
|
+
no parent dir (`CLAUDE.md`, `README.md`) has no domain, so the field is now
|
|
282
|
+
omitted instead of set to the filename (`domain: CLAUDE.md`).
|
|
283
|
+
|
|
284
|
+
### Added
|
|
285
|
+
|
|
286
|
+
- **`[enrich.keys]` — write into your own frontmatter schema.** Rename the output
|
|
287
|
+
field per kind (e.g. `description = "summary"`, `tags = "keywords"`), so `enrich`
|
|
288
|
+
fills a repo's _existing_ field names instead of imposing its own — and treats a
|
|
289
|
+
doc that already carries the renamed field as present (no duplicate). `digest`
|
|
290
|
+
reads the same renamed key for folder purposes. Defaults to the kind name, so
|
|
291
|
+
it's zero-config unless you need it.
|
|
292
|
+
|
|
293
|
+
## [0.6.0] — 2026-07-12
|
|
294
|
+
|
|
295
|
+
### Added
|
|
296
|
+
|
|
297
|
+
- **`repolens enrich`** — generate `description` + `tags` frontmatter (and a
|
|
298
|
+
one-line purpose docstring/comment for code) with a **local model**, so the
|
|
299
|
+
metadata that powers `find`/`digest` writes itself. **Bring your own model:** a
|
|
300
|
+
`[enrich]` config block sets `model` (default `llama3.2`), `endpoint` (ollama's
|
|
301
|
+
`/api/generate` shape by default), and `fields` (default `["description","tags"]`;
|
|
302
|
+
add `"domain"`, derived from the top dir, if you want it). It talks to the model
|
|
303
|
+
over **stdlib HTTP** — no Python dependency, and `find`/`lint`/`index`/`digest`
|
|
304
|
+
never touch a model. This is the **one command that writes to your source files**:
|
|
305
|
+
it only FILLS MISSING fields (never clobbers; `--force` regenerates), respects
|
|
306
|
+
`.gitignore` (same walk as the indexer), and `--dry` previews. No model server →
|
|
307
|
+
a clear message, never a crash.
|
|
308
|
+
|
|
309
|
+
## [0.5.0] — 2026-07-12
|
|
310
|
+
|
|
311
|
+
### Added
|
|
312
|
+
|
|
313
|
+
- **Incremental indexing.** `repolens index` now re-indexes only changed files —
|
|
314
|
+
a `files(relpath,size,mtime,hash)` table drives a stat-gate → blake2b hash →
|
|
315
|
+
`DELETE`/`INSERT` upsert → delete-reconcile pass, in one WAL transaction. A
|
|
316
|
+
`touch` or a fresh clone (which resets mtimes) does **not** re-index unchanged
|
|
317
|
+
content (the hash confirms). `repolens index --rebuild` is the always-correct
|
|
318
|
+
full backstop; `--optimize` compacts on demand; FTS5 auto-optimizes every ~200
|
|
319
|
+
changes. Read-path (`find`/`digest`) refreshes are now incremental.
|
|
320
|
+
- **Schema-agnostic frontmatter indexing.** A sparse EAV table
|
|
321
|
+
`frontmatter(relpath, key, value)` makes _any_ frontmatter key queryable — a doc
|
|
322
|
+
has no row for a key it lacks, so different conventions (e.g. `paths:`,
|
|
323
|
+
`name/description:`, `sector:`) coexist in one repo with **no schema imposed or
|
|
324
|
+
clobbered**. Parsed by a **total, stdlib-only** flat-frontmatter parser
|
|
325
|
+
(`frontmatter.py`) that degrades nested/malformed YAML to searchable text and
|
|
326
|
+
never raises — no `pyyaml` dependency. The flattened block stays in the FTS index
|
|
327
|
+
for full-text.
|
|
328
|
+
|
|
329
|
+
### Changed
|
|
330
|
+
|
|
331
|
+
- **Richer `digest`.** The session-start map now lists **root folders each with a
|
|
332
|
+
one-line purpose** (from a folder's `description` frontmatter or its README/H1)
|
|
333
|
+
and the **database name with every table grouped by prefix** (`fin_*`,
|
|
334
|
+
`health_*`, … + `core` + `views`) instead of a flat list truncated at 12. A new
|
|
335
|
+
`--full` tier adds per-folder docs with their descriptions; `--max-lines`
|
|
336
|
+
(default raised to 40) stays the budget guard; it degrades gracefully with no DB
|
|
337
|
+
/ no frontmatter / no README. Richness via selection + grouping, not volume.
|
|
338
|
+
|
|
339
|
+
## [0.4.3] — 2026-07-12
|
|
340
|
+
|
|
341
|
+
### Changed
|
|
342
|
+
|
|
343
|
+
- **`repolens env` now probes tool versions concurrently** — a SessionStart hook
|
|
344
|
+
must stay fast regardless of allowlist size. Sequential probing made total time
|
|
345
|
+
the _sum_ of every tool's `--version` (a heavy one like `streamlit`, or a tool
|
|
346
|
+
whose `--version` hangs to the timeout, dominated). Now the per-tool probes run
|
|
347
|
+
in a thread pool, so wall-time is bounded by the _slowest single_ tool, and the
|
|
348
|
+
per-probe timeout is tightened to 1.5s. Order is preserved.
|
|
349
|
+
|
|
350
|
+
## [0.4.2] — 2026-07-12
|
|
351
|
+
|
|
352
|
+
### Fixed
|
|
353
|
+
|
|
354
|
+
- **`repolens env` now parses `v`-prefixed and hash-suffixed versions correctly.**
|
|
355
|
+
The version regex used a `\b\d` anchor, which has no word boundary between the
|
|
356
|
+
`v` and the digits in `node v25.8.0` — so it skipped the real version and grabbed
|
|
357
|
+
a later token (reported `node 8.0`, `duckdb 5.4`). Now anchored on "a dotted
|
|
358
|
+
number not preceded by a digit/dot," so `v25.8.0 → 25.8.0`, `v1.5.4 …hash → 1.5.4`.
|
|
359
|
+
|
|
360
|
+
## [0.4.1] — 2026-07-12
|
|
361
|
+
|
|
362
|
+
### Changed
|
|
363
|
+
|
|
364
|
+
- **`repolens init` now installs the SessionStart hook by default** — when the
|
|
365
|
+
repo is a Claude Code repo (a `.claude/` dir exists). The install is additive
|
|
366
|
+
(it integrates with any existing hooks and never clobbers them), so the fresh
|
|
367
|
+
repo map is the default payoff of `init`, not a hidden second command. In a
|
|
368
|
+
non-Claude repo, `init` writes no agent config and prints a one-line hint
|
|
369
|
+
instead — it never presumes a harness that isn't there. `init --no-hook` opts out.
|
|
370
|
+
- **The SessionStart hook now runs `repolens digest && repolens env`** — the
|
|
371
|
+
freshness pair (the repo map _and_ the real toolchain), env on by default.
|
|
372
|
+
`repolens hook --no-env` drops env. (Replaces the old `--with-env` opt-in.)
|
|
373
|
+
|
|
374
|
+
## [0.4.0] — 2026-07-12
|
|
375
|
+
|
|
376
|
+
### Changed
|
|
377
|
+
|
|
378
|
+
- **`.gitignore` is now respected by default.** The file corpus (markdown + code)
|
|
379
|
+
skips any path git ignores — so secrets, `.env`, and ignored build output stay
|
|
380
|
+
out of the index. Detected via `git ls-files -co --exclude-standard`; a repo
|
|
381
|
+
with no git (or no `git` on PATH) still indexes everything, unchanged.
|
|
382
|
+
- Set `include_gitignored = true` under `[repolens]` to index gitignored file
|
|
383
|
+
content too — the personal/knowledge-repo mode (notes you ignore but want
|
|
384
|
+
searchable). SQLite **schema** discovery is unaffected either way: table/column
|
|
385
|
+
names are indexed regardless of gitignore (names only, opt-in via
|
|
386
|
+
`[integrations.sqlite]`), since databases are usually — and safely — ignored.
|
|
387
|
+
|
|
388
|
+
## [0.3.0] — 2026-07-12
|
|
389
|
+
|
|
390
|
+
### Added
|
|
391
|
+
|
|
392
|
+
- **`repolens digest`** — a compact, budgeted (`--max-lines`) repo map read from the
|
|
393
|
+
index (name, what's indexed, busiest dirs, DB tables, and a `find`-vs-`rg` routing
|
|
394
|
+
pointer), for injecting at an agent's session start. Orientation, never a dump.
|
|
395
|
+
- **`repolens env`** — an OS-aware, present-only toolchain probe: the OS plus the
|
|
396
|
+
installed tools (with versions) from a `[env].tools` allowlist that `init`
|
|
397
|
+
auto-seeds from the repo's manifests (`pyproject`→python, `package.json`→node, …).
|
|
398
|
+
Robust version probe (timeout, both streams, present-without-version fallback).
|
|
399
|
+
- **`repolens hook`** — prints a Claude Code SessionStart-hook snippet by default;
|
|
400
|
+
`--install` **additively** merges it into the repo's `.claude/settings.json`
|
|
401
|
+
(never clobbers an existing hook or key, idempotent); `--check` dry-runs.
|
|
402
|
+
- README repositioned as "the agent-context freshness layer" (own-your-context;
|
|
403
|
+
judgment-not-state; agent-agnostic; a `find` vs `rg` routing section).
|
|
404
|
+
|
|
405
|
+
## [0.2.0] — 2026-07-12
|
|
406
|
+
|
|
407
|
+
### Added
|
|
408
|
+
|
|
409
|
+
- **SQLite auto-discovery** on `repolens init` — scans for `*.db` / `*.sqlite` /
|
|
410
|
+
`*.sqlite3` (including gitignored files, skipping `*.bak*` backups and the
|
|
411
|
+
index cache), validates each is a real SQLite database, and writes the ones it
|
|
412
|
+
finds into `[integrations.sqlite]` automatically. Schema-only + read-only (only
|
|
413
|
+
table/column names, never row data). `repolens init --no-db` opts out.
|
|
414
|
+
- **Multiple databases** — `[integrations.sqlite]` now accepts a `paths` list;
|
|
415
|
+
the index covers every configured DB's table/column names.
|
|
416
|
+
|
|
417
|
+
### Changed
|
|
418
|
+
|
|
419
|
+
- `[integrations.sqlite]` is now a `paths` list. The legacy singular
|
|
420
|
+
`path = "..."` is still accepted (parsed into a one-item list) — no action
|
|
421
|
+
needed on upgrade.
|
|
422
|
+
|
|
423
|
+
## [0.1.0] — 2026-07-11
|
|
424
|
+
|
|
425
|
+
### Added
|
|
426
|
+
|
|
427
|
+
- Initial release.
|
|
428
|
+
- `repolens find` — ranked (BM25) search over markdown (full text), code/config
|
|
429
|
+
(by purpose line), and, optionally, SQLite table names; includes gitignored
|
|
430
|
+
content. Plain-`LIKE` fallback (announced) when FTS5 is unavailable.
|
|
431
|
+
- `repolens lint` — structural corpus checks (dead links, empty files, malformed
|
|
432
|
+
frontmatter, duplicate titles, staleness) plus config-declared per-type field
|
|
433
|
+
requirements. Enforced by a bundled pre-commit hook (`repolens lint --strict`).
|
|
434
|
+
- `repolens index` — build the disposable, gitignored SQLite index (atomic
|
|
435
|
+
rebuild via a temp file + `os.replace`). Full rebuild (incremental is on the
|
|
436
|
+
roadmap for v0.2).
|
|
437
|
+
- `repolens init` — scaffold `.repometa.toml`, gitignore the index cache, and
|
|
438
|
+
install the pre-commit hook.
|
|
439
|
+
- Config-driven throughout (`.repometa.toml`): repo-root marker, folder→type
|
|
440
|
+
classification with `recursive`/`exclude`/`require`, frontmatter `type:`
|
|
441
|
+
override, configurable skip lists and code extensions, and an optional
|
|
442
|
+
off-by-default SQLite integration.
|
|
443
|
+
- Stdlib-only; Python 3.11+.
|
|
444
|
+
|
|
445
|
+
[Unreleased]: https://github.com/hawkesj12/repolens/compare/v0.7.2...HEAD
|
|
446
|
+
[0.7.2]: https://github.com/hawkesj12/repolens/compare/v0.7.1...v0.7.2
|
|
447
|
+
[0.7.1]: https://github.com/hawkesj12/repolens/compare/v0.7.0...v0.7.1
|
|
448
|
+
[0.7.0]: https://github.com/hawkesj12/repolens/compare/v0.6.1...v0.7.0
|
|
449
|
+
[0.6.1]: https://github.com/hawkesj12/repolens/compare/v0.6.0...v0.6.1
|
|
450
|
+
[0.6.0]: https://github.com/hawkesj12/repolens/compare/v0.5.0...v0.6.0
|
|
451
|
+
[0.5.0]: https://github.com/hawkesj12/repolens/compare/v0.4.3...v0.5.0
|
|
452
|
+
[0.4.3]: https://github.com/hawkesj12/repolens/compare/v0.4.2...v0.4.3
|
|
453
|
+
[0.4.2]: https://github.com/hawkesj12/repolens/compare/v0.4.1...v0.4.2
|
|
454
|
+
[0.4.1]: https://github.com/hawkesj12/repolens/compare/v0.4.0...v0.4.1
|
|
455
|
+
[0.4.0]: https://github.com/hawkesj12/repolens/compare/v0.3.0...v0.4.0
|
|
456
|
+
[0.3.0]: https://github.com/hawkesj12/repolens/compare/v0.2.0...v0.3.0
|
|
457
|
+
[0.2.0]: https://github.com/hawkesj12/repolens/compare/v0.1.0...v0.2.0
|
|
458
|
+
[0.1.0]: https://github.com/hawkesj12/repolens/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Justin Hawkes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|