kernel-lore-mcp 0.3.3__tar.gz → 0.3.4__tar.gz

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/CHANGELOG.md

@@ -10,6 +10,33 @@ release tags move them into a dated section. Release process in
 
 ## [Unreleased]
 
+## [0.3.4] - 2026-04-23
+
+### Added
+
+- Added `kernel-lore-reindex`, a dedicated derived-tier rebuild command
+  for BM25, TID parquet, and path vocabulary refreshes after live sync.
+- Added tier generation and freshness reporting to `/status`,
+  `kernel-lore-mcp status`, coverage resources, and corpus stats so
+  operators can tell exactly which derived indexes are stale.
+
+### Changed
+
+- `kernel-lore-sync` is now live-safe by default: it fetches and
+  ingests lore shards without automatically rebuilding expensive
+  derived tiers on the serving box.
+- Derived rebuilds are now explicit via `kernel-lore-reindex` or the
+  opt-in sync flags `--with-tid-rebuild`, `--with-path-vocab-rebuild`,
+  and `--with-derived-rebuilds`.
+
+### Fixed
+
+- Sync no longer bumps the corpus generation, BM25 marker, TID marker,
+  or path-vocabulary marker when no corresponding content was actually
+  ingested or rebuilt.
+- BM25 and path-mention readers now reject stale derived indexes instead
+  of silently serving results from a previous corpus generation.
+
 ## [0.3.2] — 2026-04-22
 
 ### Added

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/Cargo.lock

@@ -2638,7 +2638,7 @@ dependencies = [
 
 [[package]]
 name = "kernel_lore_mcp"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "aho-corasick",
  "anyhow",

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kernel_lore_mcp"
-version = "0.3.3"
+version = "0.3.4"
 edition = "2024"
 rust-version = "1.85"
 authors = ["Michael Bommarito <michael@bommaritollc.com>"]
@@ -145,7 +145,7 @@ version = "0.3"
 features = ["env-filter", "json"]
 
 [[bin]]
-name = "reindex"
+name = "kernel-lore-reindex"
 path = "src/bin/reindex.rs"
 
 [[bin]]

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kernel-lore-mcp
-Version: 0.3.3
+Version: 0.3.4
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kernel-lore-mcp"
-version = "0.3.3"
+version = "0.3.4"
 description = "MCP server exposing fast, structured search over lore.kernel.org to LLM developer tools."
 readme = "README.md"
 license = { text = "MIT" }
@@ -59,6 +59,9 @@ kernel-lore-ingest = "kernel_lore_mcp.cli.ingest:main"
 # generation bump). This thin Python entry point exec's the Rust
 # binary so `uv tool install`d users get it without hunting PATH.
 kernel-lore-sync = "kernel_lore_mcp.cli.sync:main"
+# kernel-lore-reindex rebuilds slower derived tiers from the local
+# corpus without refetching lore. Complements the live-safe sync path.
+kernel-lore-reindex = "kernel_lore_mcp.cli.reindex:main"
 # Health / repair probe for local shard mirrors and tier files.
 kernel-lore-doctor = "kernel_lore_mcp.cli.doctor:main"
 

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/src/bin/ingest.rs

@@ -265,9 +265,16 @@ fn main() -> Result<()> {
             "over.generation marker NOT advanced: readers will bypass over.db until reconciled"
         );
     }
-    state
-        .set_tier_generation("bm25", new_gen)
-        .context("set bm25.generation marker")?;
+    if !skip_bm25 {
+        state
+            .set_tier_generation("bm25", new_gen)
+            .context("set bm25.generation marker")?;
+    } else {
+        tracing::warn!(
+            corpus_gen = new_gen,
+            "bm25.generation marker NOT advanced: BM25 was deferred for this ingest"
+        );
+    }
     state
         .set_tier_generation("trigram", new_gen)
         .context("set trigram.generation marker")?;

kernel_lore_mcp-0.3.4/src/bin/reindex.rs

@@ -0,0 +1,213 @@
+//! `kernel-lore-reindex` — rebuild derived tiers from the local data
+//! store without refetching lore.
+//!
+//! This is the maintenance complement to the live-safe sync path:
+//! `kernel-lore-sync` keeps raw metadata, over.db, and optional BM25
+//! current; `kernel-lore-reindex` rebuilds slower derived artifacts
+//! from the already-downloaded corpus.
+//!
+//! Defaults:
+//!   * `tid`
+//!   * `path_vocab`
+//!
+//! Explicit `--tier bm25` is supported, but a long-lived serving
+//! process may need a restart (or a later generation bump) before it
+//! reloads the rebuilt Tantivy reader.
+
+use std::path::PathBuf;
+use std::time::Instant;
+
+use anyhow::{Context, Result, anyhow};
+
+const REINDEX_VERSION: &str = env!("CARGO_PKG_VERSION");
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Tier {
+    Tid,
+    PathVocab,
+    Bm25,
+}
+
+impl Tier {
+    fn parse(raw: &str) -> Result<Self> {
+        match raw {
+            "tid" => Ok(Self::Tid),
+            "path_vocab" | "path-vocab" | "paths" => Ok(Self::PathVocab),
+            "bm25" => Ok(Self::Bm25),
+            other => Err(anyhow!(
+                "unknown tier {other:?}; use tid | path_vocab | bm25"
+            )),
+        }
+    }
+
+    fn name(self) -> &'static str {
+        match self {
+            Self::Tid => "tid",
+            Self::PathVocab => "path_vocab",
+            Self::Bm25 => "bm25",
+        }
+    }
+}
+
+#[derive(Default)]
+struct Args {
+    data_dir: Option<PathBuf>,
+    tiers: Vec<String>,
+    all: bool,
+}
+
+fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .with_writer(std::io::stderr)
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
+        )
+        .json()
+        .init();
+
+    let args = parse_args()?;
+    let tiers = resolve_tiers(&args)?;
+    let data_dir = args
+        .data_dir
+        .or_else(|| std::env::var_os("KLMCP_DATA_DIR").map(PathBuf::from))
+        .context("--data-dir or KLMCP_DATA_DIR required")?;
+    std::fs::create_dir_all(&data_dir)?;
+    let tier_names: Vec<&str> = tiers.iter().map(|t| t.name()).collect();
+
+    let state = _core::State::new(&data_dir)?;
+    let _writer_lock = state
+        .acquire_writer_lock()
+        .context("another writer is running (writer.lock held)")?;
+    let corpus_generation = state.generation().context("read corpus generation")?;
+
+    tracing::info!(
+        version = REINDEX_VERSION,
+        data_dir = %data_dir.display(),
+        corpus_generation,
+        tiers = ?tier_names,
+        "reindex starting"
+    );
+
+    let start = Instant::now();
+    for tier in tiers {
+        let tier_start = Instant::now();
+        match tier {
+            Tier::Tid => {
+                tracing::info!("tid rebuild starting");
+                let (path, rows) =
+                    _core::rebuild_tid(&data_dir).context("rebuild tid side-table")?;
+                state
+                    .set_tier_generation("tid", corpus_generation)
+                    .context("set tid.generation marker")?;
+                tracing::info!(
+                    rows,
+                    marker_generation = corpus_generation,
+                    path = %path.display(),
+                    elapsed_secs = tier_start.elapsed().as_secs_f64(),
+                    "tid rebuild done"
+                );
+            }
+            Tier::PathVocab => {
+                tracing::info!("path vocab rebuild starting");
+                let count = _core::path_tier::rebuild_vocab_from_over(&data_dir)
+                    .context("rebuild path vocab from over/parquet")?;
+                if count > 0 {
+                    state
+                        .set_tier_generation("path_vocab", corpus_generation)
+                        .context("set path_vocab.generation marker")?;
+                    tracing::info!(
+                        paths = count,
+                        marker_generation = corpus_generation,
+                        path = %data_dir.join("paths").join("vocab.txt").display(),
+                        elapsed_secs = tier_start.elapsed().as_secs_f64(),
+                        "path vocab rebuild done"
+                    );
+                } else {
+                    tracing::info!(
+                        paths = count,
+                        elapsed_secs = tier_start.elapsed().as_secs_f64(),
+                        "path vocab rebuild produced an empty vocab; marker left unchanged"
+                    );
+                }
+            }
+            Tier::Bm25 => {
+                tracing::warn!(
+                    "bm25 rebuild starting; restart serving processes or run a later generation bump if you need live readers to reload immediately"
+                );
+                let docs = _core::rebuild_bm25(&data_dir).context("rebuild bm25 from store")?;
+                state
+                    .set_tier_generation("bm25", corpus_generation)
+                    .context("set bm25.generation marker")?;
+                tracing::warn!(
+                    docs,
+                    marker_generation = corpus_generation,
+                    elapsed_secs = tier_start.elapsed().as_secs_f64(),
+                    "bm25 rebuild done; marker updated, but long-lived readers may still need a restart to pick up the new index"
+                );
+            }
+        }
+    }
+
+    tracing::info!(
+        elapsed_secs = start.elapsed().as_secs_f64(),
+        corpus_generation,
+        tiers = ?tier_names,
+        "reindex complete"
+    );
+    Ok(())
+}
+
+fn resolve_tiers(args: &Args) -> Result<Vec<Tier>> {
+    let mut out: Vec<Tier> = Vec::new();
+    if args.all {
+        out.extend([Tier::Tid, Tier::PathVocab, Tier::Bm25]);
+    }
+    if !args.all && args.tiers.is_empty() {
+        out.extend([Tier::Tid, Tier::PathVocab]);
+    }
+    for raw in &args.tiers {
+        let tier = Tier::parse(raw)?;
+        if !out.contains(&tier) {
+            out.push(tier);
+        }
+    }
+    Ok(out)
+}
+
+fn parse_args() -> Result<Args> {
+    let mut args = Args::default();
+    let mut it = std::env::args().skip(1);
+    while let Some(a) = it.next() {
+        match a.as_str() {
+            "--data-dir" => args.data_dir = it.next().map(PathBuf::from),
+            "--tier" => args.tiers.push(it.next().context("--tier expects a name")?),
+            "--all" => args.all = true,
+            "--help" | "-h" => {
+                println!(
+                    "kernel-lore-reindex\n\
+                     version: {REINDEX_VERSION}\n\
+                     \n\
+                     --data-dir PATH       (or $KLMCP_DATA_DIR)\n\
+                     --tier NAME           repeatable: tid | path_vocab | bm25\n\
+                     --all                 rebuild tid + path_vocab + bm25\n\
+                     \n\
+                     Defaults:\n\
+                     with no --tier flags, rebuilds tid + path_vocab only.\n\
+                     \n\
+                     Notes:\n\
+                     - This command rebuilds from the local corpus; it does NOT refetch lore.\n\
+                     - `--tier bm25` updates the on-disk index and marker, but a long-lived\n\
+                       serving process may need a restart before it reloads the rebuilt reader.\n"
+                );
+                std::process::exit(0);
+            }
+            "--version" | "-V" => {
+                println!("{REINDEX_VERSION}");
+                std::process::exit(0);
+            }
+            other => return Err(anyhow!("unknown arg: {other}")),
+        }
+    }
+    Ok(args)
+}

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/src/bin/sync.rs

@@ -12,8 +12,9 @@
 //!   3. For each changed shard, gix clone-or-fetch in parallel.
 //!   4. For each fetched shard, ingest via the existing
 //!      `ingest_shard_with_bm25` under a single writer lock.
-//!   5. Rebuild tid side-table once at the end, bump generation once.
-//!   6. Persist the fresh manifest cache (only on full success, so a
+//!   5. Optionally rebuild derived tiers (tid / path vocab) at the end.
+//!   6. Bump generation once iff the raw corpus changed.
+//!   7. Persist the fresh manifest cache (only on full success, so a
 //!      partial-failure rerun re-fetches the same shards).
 //!
 //! Exit codes:
@@ -28,7 +29,8 @@
 //!   kernel-lore-sync --dry-run  # manifest fetch + diff only
 //!
 //! Stays single-process: the writer lock taken here covers manifest
-//! fetch, gix fetch, ingest, tid rebuild, and the generation bump,
+//! fetch, gix fetch, ingest, optional derived rebuilds, and the
+//! generation bump,
 //! so a concurrent sync invocation fails the flock and exits cleanly
 //! without touching state.
 
@@ -127,6 +129,8 @@ struct SyncStatusRecord {
     stage: String,
     with_bm25: bool,
     with_over: bool,
+    rebuild_tid: bool,
+    rebuild_path_vocab: bool,
     workers: usize,
     total_shards: usize,
     started_shards: usize,
@@ -159,6 +163,8 @@ impl SyncStateReporter {
         run_id: &str,
         with_bm25: bool,
         with_over: bool,
+        rebuild_tid: bool,
+        rebuild_path_vocab: bool,
         workers: usize,
         total_shards: usize,
     ) -> Result<Self> {
@@ -173,6 +179,8 @@ impl SyncStateReporter {
                 stage: "starting".to_string(),
                 with_bm25,
                 with_over,
+                rebuild_tid,
+                rebuild_path_vocab,
                 workers,
                 total_shards,
                 started_shards: 0,
@@ -521,6 +529,13 @@ fn main() -> Result<()> {
         (false, true) => false,
         (false, false) => over_path.exists(),
     };
+    let rebuild_tid = args.with_derived_rebuilds || args.with_tid_rebuild;
+    let rebuild_path_vocab = args.with_derived_rebuilds || args.with_path_vocab_rebuild;
+    if !rebuild_tid && !rebuild_path_vocab {
+        tracing::info!(
+            "derived rebuilds deferred (live-safe default). Run `kernel-lore-reindex` or pass explicit --with-*-rebuild flags when you want tid/path vocab refreshed inline."
+        );
+    }
 
     // Resolve manifest paths to ChangedShard descriptors.
     let changed: Vec<ChangedShard> = changed_paths
@@ -538,6 +553,8 @@ fn main() -> Result<()> {
         &run_id,
         !skip_bm25,
         with_over,
+        rebuild_tid,
+        rebuild_path_vocab,
         workers,
         changed.len(),
     )
@@ -674,6 +691,8 @@ fn main() -> Result<()> {
         lists = stores.len(),
         with_over,
         with_bm25 = !skip_bm25,
+        rebuild_tid,
+        rebuild_path_vocab,
         workers = workers,
         "ingest phase starting"
     );
@@ -782,33 +801,46 @@ fn main() -> Result<()> {
         drop(stage_guard);
     }
 
-    // tid rebuild: reasonable to do every sync tick; touches only the
-    // subset of rows whose tid changed. Cheap at steady state.
-    sync_state
-        .set_stage("tid_rebuild", None)
-        .context("update sync status to tid_rebuild")?;
-    let tid_stage_guard = stage_heartbeat(sync_state.clone(), "tid_rebuild", None, start);
-    tracing::info!("tid rebuild starting");
-    let tid_result = _core::rebuild_tid(&data_dir).context("rebuild tid side-table")?;
-    tracing::info!(
-        rows = tid_result.1,
-        path = %tid_result.0.display(),
-        "tid rebuild done"
-    );
-    drop(tid_stage_guard);
-    sync_state
-        .set_stage("path_vocab_rebuild", None)
-        .context("update sync status to path_vocab_rebuild")?;
-    let path_stage_guard = stage_heartbeat(sync_state.clone(), "path_vocab_rebuild", None, start);
-    tracing::info!("path vocab rebuild starting");
-    if let Some(path_count) = rebuild_path_vocab_after_sync(&data_dir)? {
+    let mut tid_rebuilt = false;
+    let mut path_vocab_rebuilt = false;
+    if rebuild_tid {
+        sync_state
+            .set_stage("tid_rebuild", None)
+            .context("update sync status to tid_rebuild")?;
+        let tid_stage_guard = stage_heartbeat(sync_state.clone(), "tid_rebuild", None, start);
+        tracing::info!("tid rebuild starting");
+        let tid_result = _core::rebuild_tid(&data_dir).context("rebuild tid side-table")?;
         tracing::info!(
-            paths = path_count,
-            path = %data_dir.join("paths").join("vocab.txt").display(),
-            "path vocab rebuild done"
+            rows = tid_result.1,
+            path = %tid_result.0.display(),
+            "tid rebuild done"
         );
+        tid_rebuilt = true;
+        drop(tid_stage_guard);
+    } else {
+        tracing::info!("tid rebuild skipped (live-safe default)");
+    }
+    if rebuild_path_vocab {
+        sync_state
+            .set_stage("path_vocab_rebuild", None)
+            .context("update sync status to path_vocab_rebuild")?;
+        let path_stage_guard =
+            stage_heartbeat(sync_state.clone(), "path_vocab_rebuild", None, start);
+        tracing::info!("path vocab rebuild starting");
+        if let Some(path_count) = rebuild_path_vocab_after_sync(&data_dir)? {
+            tracing::info!(
+                paths = path_count,
+                path = %data_dir.join("paths").join("vocab.txt").display(),
+                "path vocab rebuild done"
+            );
+            path_vocab_rebuilt = true;
+        } else {
+            tracing::info!("path vocab rebuild skipped by environment");
+        }
+        drop(path_stage_guard);
+    } else {
+        tracing::info!("path vocab rebuild skipped (live-safe default)");
     }
-    drop(path_stage_guard);
 
     // Tally.
     let mut total_ingested: u64 = 0;
@@ -851,36 +883,73 @@ fn main() -> Result<()> {
         }
     }
 
-    sync_state
-        .set_stage("generation_bump", None)
-        .context("update sync status to generation_bump")?;
-    tracing::info!("generation bump starting");
-    let new_gen = state.bump_generation().context("bump generation")?;
-    tracing::info!(generation = new_gen, "generation bumped");
+    let corpus_changed = total_ingested > 0;
+    let current_gen = state.generation().context("read generation")?;
+    let new_gen = if corpus_changed {
+        sync_state
+            .set_stage("generation_bump", None)
+            .context("update sync status to generation_bump")?;
+        tracing::info!("generation bump starting");
+        let bumped = state.bump_generation().context("bump generation")?;
+        tracing::info!(generation = bumped, "generation bumped");
+        bumped
+    } else {
+        tracing::info!(
+            generation = current_gen,
+            "generation not bumped (no new messages ingested)"
+        );
+        current_gen
+    };
 
     // Per-tier markers. Same discipline as bin/ingest.rs: only
     // advance the `over` marker on full success so readers bypass
     // over.db on drift.
-    if with_over && total_over_failed == 0 {
+    if corpus_changed && with_over && total_over_failed == 0 {
         state
             .set_tier_generation("over", new_gen)
             .context("set over.generation marker")?;
-    } else if with_over {
+    } else if corpus_changed && with_over {
         tracing::warn!(
             over_failed_shards = total_over_failed,
             corpus_gen = new_gen,
             "over.generation marker NOT advanced"
         );
     }
-    state
-        .set_tier_generation("bm25", new_gen)
-        .context("set bm25.generation marker")?;
-    state
-        .set_tier_generation("trigram", new_gen)
-        .context("set trigram.generation marker")?;
-    state
-        .set_tier_generation("tid", new_gen)
-        .context("set tid.generation marker")?;
+    if corpus_changed && !skip_bm25 {
+        state
+            .set_tier_generation("bm25", new_gen)
+            .context("set bm25.generation marker")?;
+    } else if corpus_changed && skip_bm25 {
+        tracing::warn!(
+            corpus_gen = new_gen,
+            "bm25.generation marker NOT advanced: BM25 was deferred for this sync"
+        );
+    }
+    if corpus_changed {
+        state
+            .set_tier_generation("trigram", new_gen)
+            .context("set trigram.generation marker")?;
+    }
+    if tid_rebuilt {
+        state
+            .set_tier_generation("tid", new_gen)
+            .context("set tid.generation marker")?;
+    } else if corpus_changed {
+        tracing::warn!(
+            corpus_gen = new_gen,
+            "tid.generation marker NOT advanced: tid rebuild was deferred for this sync"
+        );
+    }
+    if path_vocab_rebuilt {
+        state
+            .set_tier_generation("path_vocab", new_gen)
+            .context("set path_vocab.generation marker")?;
+    } else if corpus_changed {
+        tracing::warn!(
+            corpus_gen = new_gen,
+            "path_vocab.generation marker NOT advanced: path vocab rebuild was deferred for this sync"
+        );
+    }
 
     // Persist manifest cache — start from the existing local entries
     // (so filtered-out shards keep their history) and overwrite only
@@ -974,6 +1043,9 @@ struct Args {
     with_bm25: bool,
     with_over: bool,
     no_over: bool,
+    with_tid_rebuild: bool,
+    with_path_vocab_rebuild: bool,
+    with_derived_rebuilds: bool,
     max_retries: u32,
     dry_run: bool,
 }
@@ -1005,6 +1077,9 @@ fn parse_args() -> Result<Args> {
             "--with-bm25" => args.with_bm25 = true,
             "--with-over" => args.with_over = true,
             "--no-over" => args.no_over = true,
+            "--with-tid-rebuild" => args.with_tid_rebuild = true,
+            "--with-path-vocab-rebuild" => args.with_path_vocab_rebuild = true,
+            "--with-derived-rebuilds" => args.with_derived_rebuilds = true,
             "--max-retries" => {
                 args.max_retries = it.next().and_then(|s| s.parse().ok()).unwrap_or(3);
             }
@@ -1031,6 +1106,14 @@ fn parse_args() -> Result<Args> {
                      --with-over           force on over.db writes\n\
                      --no-over             force off over.db writes\n\
                                            (default: on iff <data_dir>/over.db exists)\n\
+                     --with-tid-rebuild    rebuild tid side-table inline\n\
+                                           after ingest (heavier)\n\
+                     --with-path-vocab-rebuild\n\
+                                           rebuild paths/vocab.txt inline\n\
+                                           after ingest (heavier)\n\
+                     --with-derived-rebuilds\n\
+                                           rebuild both tid + path vocab\n\
+                                           inline after ingest\n\
                      --max-retries N       per-shard ingest retry count (default: 3)\n\
                      --dry-run             fetch manifest + diff, don't touch shards\n\
                      \n\
@@ -1043,7 +1126,7 @@ fn parse_args() -> Result<Args> {
                                                   (default: 1)\n\
                      KLMCP_BM25_WRITER_MEMORY_MB BM25 total writer memory budget\n\
                                                   in MiB (default: 128)\n\
-                     KLMCP_SKIP_PATH_VOCAB=1     skip automatic paths/vocab.txt rebuild\n"
+                     KLMCP_SKIP_PATH_VOCAB=1     skip requested path vocab rebuild\n"
                 );
                 std::process::exit(0);
             }

{kernel_lore_mcp-0.3.3 → kernel_lore_mcp-0.3.4}/src/kernel_lore_mcp/__init__.py

@@ -15,7 +15,7 @@ from typing import Any
 
 __all__ = ["__version__", "native_version"]
 
-__version__ = "0.3.3"
+__version__ = "0.3.4"
 
 
 def __getattr__(name: str) -> Any: