lorme 0.1.1__py3-none-any.whl

lorme/__init__.py

@@ -0,0 +1,10 @@
+"""lorme — Local Offline Retrieval & MCP Engine.
+
+See docs/architecture.md for system design and docs/configuration.md
+for environment variables.
+"""
+
+try:
+    from lorme._version import __version__
+except ImportError:
+    __version__ = "0.0.0.dev0+unknown"

lorme/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.1'
+__version_tuple__ = version_tuple = (0, 1, 1)
+
+__commit_id__ = commit_id = None

lorme/bootstrap.yaml

@@ -0,0 +1,89 @@
+# lorme configuration
+# Generated by: lorme init
+# All values shown are defaults. Uncomment to override.
+# Without this file, lorme works with XDG defaults.
+# Docs: docs/configuration.md
+
+# === Database ===
+
+# Common: base directory for all collections
+# database:
+#   dir: ~/.local/share/lorme/
+#   default_collection: default
+
+# === Embedding ===
+
+# Common: model name (references llm registry entry below)
+# embedding:
+#   model: nomic-embed
+
+# Advanced
+#   mode: api              # api | builtin:gpu | builtin:cpu
+
+# === Chunking ===
+
+# Common
+# chunking:
+#   size: 1024
+
+# Advanced
+#   overlap: 128
+
+# === Search ===
+
+# Common
+# search:
+#   top_k: 5
+
+# Advanced
+#   reranking_model: ""     # llm registry entry name, empty = no reranking
+#   mmr: false              # Maximal Marginal Relevance for diversity
+#   window_size: 0          # adjacent chunk expansion, 0 = disabled
+#   max_per_source: 0       # max chunks per source, 0 = no limit
+
+# === add_source defaults ===
+
+# Common
+# add_source:
+#   preprocess: true
+
+# Advanced
+#   enrich: []              # [context, qa, meta]
+#   optimize: false
+
+# === Enrichment ===
+
+# Advanced
+# enrich:
+#   techniques: []          # [context, qa, meta, stt_correction]
+#   models: []              # llm registry entry names
+#   prompts_file: ""        # custom prompts.yaml path
+
+# === Parse ===
+
+# Advanced
+# parse:
+#   ocr_engine: ""          # tesseract | ""
+#   ocr_lang: []            # [eng, fra]
+#   stt_model: ""           # llm registry entry name
+#   video_frame_strategy: scene  # scene | fixed | hybrid | ocr
+#   frame_interval: 30      # seconds between frames (fixed/hybrid)
+#   scene_threshold: 0.3    # scene change sensitivity
+
+# === LLM Registry ===
+
+# Define inference services. Referenced by name in other sections.
+# llm:
+#   - name: nomic-embed
+#     model: nomic-ai/nomic-embed-text-v2-moe
+#     api_url: http://127.0.0.1:8082/v1/embeddings
+#     resource: local_gpu   # local_gpu | local_cpu | remote
+#
+#     # Advanced service lifecycle (must be self-contained)
+#     # start: mkdir -p /path/to/logs; podman rm -f svc 2>/dev/null; podman run -d --name svc ...
+#     # stop: podman stop svc; podman rm -f svc
+#     # start_timeout: 120
+#     # timeout: 180
+#     # concurrency: 1
+#     # batch_size: 32
+#     # verify_ssl: true

lorme/build.py

@@ -0,0 +1,334 @@
+"""Build workflow: manifest + models → optimized .db + metadata. See docs/architecture.md."""
+
+import json
+import logging
+from pathlib import Path
+
+from lorme.embedder import Embedder
+from lorme.eval import run_optimize
+from lorme.ingest import ingest_with_manifest
+from lorme.recipe import parse_recipe
+from lorme.metadata import generate_all
+from lorme.preprocess import preprocess_sources
+from lorme.store import ChunkStore
+
+logger = logging.getLogger(__name__)
+
+QUESTIONS_FILE = "questions.json"
+SCORES_FILE = "scores.jsonl"
+
+
+def validate_models(
+    configs: list[dict],
+    embedders: dict | None = None,
+) -> list[str]:
+    """Validate all model configs before starting. Returns list of errors."""
+    from lorme.embedder import _probe_api
+
+    errors = []
+    for cfg in configs:
+        name = cfg["name"]
+        mode = cfg.get("mode", "builtin")
+
+        if mode == "api":
+            url = cfg.get("api_url")
+            if not url:
+                errors.append(f"{name}: mode=api but no api_url")
+                continue
+            if not _probe_api(url, cfg.get("api_model", name), verify=False):
+                errors.append(f"{name}: API endpoint unreachable ({url})")
+        elif embedders and name in embedders:
+            pass
+        else:
+            cache_path = Path.home() / ".cache" / "huggingface" / "hub" / f"models--{name.replace('/', '--')}"
+            if not cache_path.exists():
+                errors.append(f"{name}: not in HuggingFace cache, use --allow-download")
+
+    return errors
+
+
+def _start_embedders(config) -> dict:
+    """Create embedders from config, starting services if needed. See E12.68."""
+    from lorme.embedder import create_embedder
+
+    if config.embedding_models:
+        return {
+            emb_cfg["name"]: create_embedder(config, emb_cfg)
+            for emb_cfg in config.embedding_models
+        }
+
+    entry = config.get_embedding_entry()
+    if entry:
+        from lorme.preprocess.service import start_service
+        start_service(entry)
+
+    emb = create_embedder(config, entry)
+    return {emb.model_name: emb}
+
+
+def run_build(
+    recipe_path: str,
+    docs_dir: str,
+    output_dir: str,
+    config,
+    embedders: dict | None = None,
+    embedder: Embedder | None = None,
+) -> dict:
+    """Full build pipeline: [preprocess →] validate → optimize → index → metadata.
+
+    All params resolved from config. embedders passed as instantiated objects.
+    """
+    force = config.force
+    output_level = config.output_level
+    skip_optimize = config.skip_optimize
+    report_path = config.report_path or None
+    chunk_sizes = config.optimize_chunk_sizes
+    chunk_overlaps = config.optimize_chunk_overlaps
+    top_ks = config.optimize_top_ks
+    num_questions = config.optimize_num_questions
+
+    _build_dir = config.build_dir or output_dir
+
+    from lorme.task_manager import report_progress, check_cancelled
+
+    if config.preprocess:
+        prep_recipe_path = Path(_build_dir) / (
+            Path(recipe_path).stem + "-prep" + Path(recipe_path).suffix
+        )
+        config.preprocess_recipe_out = str(prep_recipe_path)
+        check_cancelled()
+        report_progress("Preprocessing sources")
+        preprocess_sources(recipe_path, docs_dir, config)
+        recipe_path = str(prep_recipe_path)
+        docs_dir = str(Path(_build_dir) / "prep")
+
+    # Use prep dir as docs_dir if it exists (previous preprocess run)
+    if not config.preprocess and _build_dir:
+        prep_dir = Path(_build_dir) / "prep"
+        prep_manifest = Path(_build_dir) / (
+            Path(recipe_path).stem + "-prep" + Path(recipe_path).suffix
+        )
+        if prep_dir.exists() and any(prep_dir.iterdir()):
+            docs_dir = str(prep_dir)
+            if prep_manifest.exists():
+                recipe_path = str(prep_manifest)
+
+    recipe = parse_recipe(recipe_path)
+    collection = getattr(config, "collection_override", "") or recipe["collection"]
+    output_path = Path(_build_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    if embedders is None and embedder is not None:
+        embedders = {embedder.model_name: embedder}
+    if not embedders:
+        embedders = _start_embedders(config)
+
+    work_path = output_path / ".work"
+    work_path.mkdir(parents=True, exist_ok=True)
+
+    from lorme.progress import ProgressReporter
+    import time as _time
+
+    build_reporter = ProgressReporter(
+        collection=collection,
+        models=list(embedders.keys()),
+        total_configs=0,
+        level=output_level,
+    )
+    build_start = _time.time()
+
+    resumed = False
+    optimization = None
+    winning_model = next(iter(embedders))
+    if skip_optimize:
+        winning_chunk_size = config.chunk_size
+        winning_chunk_overlap = config.chunk_overlap
+    else:
+        winning_chunk_size = chunk_sizes[0] if chunk_sizes else 1024
+        winning_chunk_overlap = chunk_overlaps[0] if chunk_overlaps else 128
+
+    if not skip_optimize:
+        check_cancelled()
+        report_progress("Optimizing chunking parameters")
+        optimization = _run_optimization(
+            recipe_path=recipe_path,
+            docs_dir=docs_dir,
+            embedders=embedders,
+            work_dir=str(work_path),
+            force=force,
+            output_level=output_level,
+            config=config,
+        )
+        resumed = optimization.get("resumed", False)
+        best = optimization.get("best", {})
+        if best:
+            winning_model = best.get("model_name", winning_model)
+            winning_chunk_size = best.get("chunk_size", winning_chunk_size)
+            winning_chunk_overlap = best.get("chunk_overlap", winning_chunk_overlap)
+
+    for emb in embedders.values():
+        emb.unload()
+
+    build_reporter.print_section(
+        f"Final build — {winning_model} chunk={winning_chunk_size}/{winning_chunk_overlap}"
+    )
+
+    final_db = str(output_path / f"{collection}.db")
+    final_emb = embedders[winning_model]
+
+    t0 = _time.time()
+    cache_valid = False
+    if not force and Path(final_db).exists():
+        try:
+            _store = ChunkStore(final_db)
+            _meta = _store.get_meta()
+            _src = _store.list_sources()
+            _store.close()
+            cache_valid = (
+                len(_src) > 0
+                and _meta.get("chunk_size") == str(winning_chunk_size)
+                and _meta.get("chunk_overlap") == str(winning_chunk_overlap)
+                and _meta.get("model_name") == winning_model
+            )
+        except Exception:
+            cache_valid = False
+
+    if cache_valid:
+        build_reporter.print_step("Skipped (cached)")
+        report_progress("Indexing skipped (cached)")
+    else:
+        if Path(final_db).exists():
+            Path(final_db).unlink()
+        check_cancelled()
+        report_progress(f"Indexing {collection} ({winning_model})")
+        ingest_with_manifest(
+            recipe_path, docs_dir, str(output_path), final_emb,
+            chunk_size=winning_chunk_size, chunk_overlap=winning_chunk_overlap,
+            collection=collection,
+        )
+        build_reporter.print_step("Indexing", elapsed=_time.time() - t0)
+
+    final_emb.unload()
+
+    t0 = _time.time()
+    check_cancelled()
+    report_progress("Generating metadata")
+    generate_all(final_db)
+    build_reporter.print_step("Metadata", elapsed=_time.time() - t0)
+
+    _final_store = ChunkStore(final_db)
+    sources = _final_store.list_sources()
+    _final_store.close()
+
+    report = {
+        "collection": collection,
+        "model_name": winning_model,
+        "chunk_size": winning_chunk_size,
+        "chunk_overlap": winning_chunk_overlap,
+        "file_count": len(sources),
+        "chunk_count": sum(s["count"] for s in sources),
+        "optimization": optimization,
+        "resumed": resumed,
+    }
+
+    json_report_path = output_path / "build-report.json"
+    json_report_path.write_text(
+        json.dumps(report, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+
+    build_reporter.print_summary(
+        files=len(sources),
+        chunks=sum(s["count"] for s in sources),
+        configs_tested=len(optimization.get("all", [])) if optimization else 0,
+        elapsed=_time.time() - build_start,
+        report_path=str(json_report_path),
+    )
+
+    if not config.keep_intermediates:
+        import shutil
+        _prep = output_path / "prep"
+        _work = output_path / ".work"
+        if _prep.exists():
+            shutil.rmtree(_prep)
+        if _work.exists():
+            shutil.rmtree(_work)
+
+    return report
+
+
+def _run_optimization(
+    recipe_path: str,
+    docs_dir: str,
+    embedders: dict,
+    work_dir: str,
+    chunk_sizes: list[int] | None = None,
+    chunk_overlaps: list[int] | None = None,
+    top_ks: list[int] | None = None,
+    num_questions: int = 50,
+    force: bool = False,
+    output_level: str = "default",
+    metrics: list[str] | None = None,
+    judge_url: str = "",
+    judge_model: str = "",
+    judge_verify_ssl: bool = True,
+    report_path: str | None = None,
+    config=None,
+) -> dict:
+    """Run optimization with resumability."""
+    if config is not None:
+        if chunk_sizes is None:
+            chunk_sizes = config.optimize_chunk_sizes
+        if chunk_overlaps is None:
+            chunk_overlaps = config.optimize_chunk_overlaps
+        if top_ks is None:
+            top_ks = config.optimize_top_ks
+        if not num_questions or num_questions == 50:
+            num_questions = config.optimize_num_questions or 50
+        if metrics is None:
+            metrics = config.optimize_metrics or None
+        if not judge_url:
+            judge_url = config.llm_api_url
+        if not judge_model:
+            judge_model = config.llm_model
+            judge_verify_ssl = config.llm_verify_ssl
+
+    work_path = Path(work_dir)
+    scores_path = work_path / SCORES_FILE
+    questions_path = work_path / QUESTIONS_FILE
+
+    existing_scores = []
+    resumed = False
+
+    if not force and scores_path.exists():
+        existing_scores = [
+            json.loads(line)
+            for line in scores_path.read_text().strip().split("\n")
+            if line.strip()
+        ]
+        resumed = True
+        logger.info("Resuming: %d existing scores", len(existing_scores))
+
+    result = run_optimize(
+        embedders=embedders,
+        db_dir=work_dir,
+        recipe_path=recipe_path,
+        docs_dir=docs_dir,
+        chunk_sizes=chunk_sizes,
+        chunk_overlaps=chunk_overlaps,
+        top_ks=top_ks,
+        num_questions=num_questions,
+        output_level=output_level,
+        metrics=metrics,
+        judge_url=judge_url,
+        judge_model=judge_model,
+        judge_verify_ssl=judge_verify_ssl,
+        report_path=report_path,
+    )
+
+    with open(scores_path, "w", encoding="utf-8") as f:
+        for entry in result.get("all", []):
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    result["resumed"] = resumed
+    return result

lorme/build_config.py

@@ -0,0 +1,58 @@
+"""Unified build configuration. See docs/architecture.md."""
+
+from dataclasses import dataclass, field
+
+import yaml
+
+
+def _read_embedding_key(data: dict, path: str) -> list:
+    """Read embedding models, reject old key names."""
+    if "models" in data or "embedding_models" in data:
+        raise ValueError(
+            f"Use 'embedding:' key (not 'models:' or 'embedding_models:') "
+            f"in {path}"
+        )
+    return data.get("embedding", [])
+
+
+@dataclass
+class BuildConfig:
+    """Unified configuration for lorme build."""
+    embedding_models: list[dict] = field(default_factory=list)
+    judge_model: str = ""
+    judge_api_url: str = ""
+    judge_verify_ssl: bool = True
+    metrics: list[str] = field(default_factory=lambda: ["score_spread", "source_diversity", "result_diversity"])
+    chunk_sizes: list[int] = field(default_factory=lambda: [512, 1024, 2048])
+    chunk_overlaps: list[int] = field(default_factory=lambda: [64, 128])
+    top_ks: list[int] = field(default_factory=lambda: [3, 5, 10])
+    num_questions: int = 50
+    default_model: str = ""
+    default_chunk_size: int = 1024
+    default_chunk_overlap: int = 128
+
+    @classmethod
+    def from_file(cls, path: str) -> "BuildConfig":
+        """Load build config from a YAML file."""
+        with open(path, encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+
+        judge = data.get("judge", {})
+        optimize = data.get("optimize", {})
+        defaults = data.get("defaults", {})
+
+        return cls(
+            embedding_models=_read_embedding_key(data, path),
+            judge_model=judge.get("model", ""),
+            judge_api_url=judge.get("api_url", ""),
+            judge_verify_ssl=judge.get("verify_ssl", True),
+            metrics=data.get("metrics", ["score_spread", "source_diversity", "result_diversity"]),
+            chunk_sizes=optimize.get("chunk_sizes", [512, 1024, 2048]),
+            chunk_overlaps=optimize.get("chunk_overlaps", [64, 128]),
+            top_ks=optimize.get("top_ks", [3, 5, 10]),
+            num_questions=optimize.get("num_questions", 50),
+            default_model=defaults.get("model", ""),
+            default_chunk_size=defaults.get("chunk_size", 1024),
+            default_chunk_overlap=defaults.get("chunk_overlap", 128),
+        )
+