contextual-engine 0.9.4__tar.gz → 0.9.6__tar.gz

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/.gitignore

@@ -21,6 +21,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+pytest.ini
 
 # Virtual environments
 .venv/
@@ -106,3 +107,6 @@ output.md
 audit_report.mduv.lock
 audit.md 
 .agent
+.cursor
+.cursor/rules
+.benchmarks

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: contextual-engine
-Version: 0.9.4
+Version: 0.9.6
 Summary: Temporal-first local code memory for AI tools via MCP
 Project-URL: Homepage, https://contextuallabs.dev
 Project-URL: Documentation, https://contextuallabs.dev/docs
@@ -125,16 +125,19 @@ Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Text Processing :: Indexing
 Classifier: Typing :: Typed
-Requires-Python: <3.14,>=3.12
+Requires-Python: <3.15,>=3.11
 Requires-Dist: blake3>=0.4.1
+Requires-Dist: datasketch>=1.6.4
 Requires-Dist: docutils>=0.21
 Requires-Dist: dulwich>=0.21
-Requires-Dist: einops>=0.7
+Requires-Dist: fastembed>=0.4.2
 Requires-Dist: fastmcp<4,>=3.2
 Requires-Dist: httpx[http2]>=0.27.0
 Requires-Dist: huggingface-hub>=0.23
@@ -153,8 +156,7 @@ Requires-Dist: pygit2>=1.15
 Requires-Dist: python-frontmatter
 Requires-Dist: rapidfuzz>=3
 Requires-Dist: rich>=13.0
-Requires-Dist: safetensors>=0.4.3
-Requires-Dist: sentence-transformers>=3.0
+Requires-Dist: setproctitle>=1.3.0
 Requires-Dist: simsimd<6,>=3.9
 Requires-Dist: sqlglot>=20.0
 Requires-Dist: starlette>=0.37
@@ -162,10 +164,7 @@ Requires-Dist: structlog>=25.4
 Requires-Dist: tantivy>=0.20.1
 Requires-Dist: tenacity>=8.0
 Requires-Dist: tiktoken>=0.5
-Requires-Dist: tokenizers<=0.23.0,>=0.19
 Requires-Dist: tomli-w>=1.0.0
-Requires-Dist: torch>=2.2
-Requires-Dist: transformers<4.60,>=4.40
 Requires-Dist: tree-sitter-language-pack>=0.7.2
 Requires-Dist: typer<1.0,>=0.9
 Requires-Dist: uvicorn>=0.27
@@ -178,9 +177,6 @@ Requires-Dist: build>=1.0; extra == 'dev'
 Requires-Dist: mypy>=1.10; extra == 'dev'
 Requires-Dist: ruff>=0.4; extra == 'dev'
 Requires-Dist: twine>=5.0; extra == 'dev'
-Provides-Extra: mps
-Requires-Dist: torch>=2.4.0; extra == 'mps'
-Requires-Dist: torchvision>=0.19.0; extra == 'mps'
 Provides-Extra: test
 Requires-Dist: anyio[trio]>=4.3.0; extra == 'test'
 Requires-Dist: freezegun>=1.5.0; extra == 'test'

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/cli/commands/doctor.py

@@ -14,7 +14,6 @@ from contextual.config.paths import (
     LOG_DIR,
     MODELS_DIR,
     embed_model_present,
-    rerank_model_present,
     workspace_config,
     workspace_db,
     workspace_logs,
@@ -85,14 +84,10 @@ async def doctor(
     # 3. Models Check — check the actual weight dirs, not MODELS_DIR (which
     # always contains cache/ after setup and would false-positive before fetch).
     embed_ok = embed_model_present()
-    rerank_ok = rerank_model_present()
-    models_ok = embed_ok  # embed is the hard requirement; reranker is search-only
-    if embed_ok and rerank_ok:
-        model_detail = "Embed + reranker weights present."
-    elif embed_ok and not rerank_ok:
-        model_detail = "Embed weights present. Reranker missing — run 'contextual fetch --rerank'."
-    else:
-        model_detail = "Embed weights missing or empty. Run 'contextual fetch'."
+    models_ok = embed_ok
+    model_detail = (
+        "Embed weights present." if embed_ok else "Embed weights missing. Run 'contextual fetch'."
+    )
     checks.append(HealthCheck(name="Models", status=models_ok, detail=model_detail))
 
     # 4. Daemon & Locks Check

contextual_engine-0.9.6/contextual/cli/commands/fetch.py

@@ -0,0 +1,158 @@
+"""Global model fetch command for Contextual.
+
+Downloads the fastembed/ONNX embedding model into ~/.contextual/models/
+(the canonical global model directory) using fastembed's cache_dir parameter.
+After download, computes and stores a BLAKE3 hash pin for integrity verification
+on every subsequent model load.
+
+The old sentence-transformers download path is fully removed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import structlog
+import typer
+
+from contextual.cli.core.decorators import register_command, require_config
+from contextual.cli.core.output import (
+    model_download_progress,
+    print_error,
+    print_fetch_complete,
+    print_fetch_start,
+)
+from contextual.config.paths import GLOBAL_CONFIG, MODELS_DIR, ONNX_PIN_FILE, VERSION_FILE
+from contextual.models.unified_embedder import _FASTEMBED_MODEL, _find_onnx_model_path
+
+_EMBED_FRIENDLY = "Nomic Embed v1.5-Q (ONNX/fastembed)"
+
+
+class FetchError(Exception):
+    """Raised when a global model fetch cannot complete."""
+
+    def __init__(self, message: str, step: str | None = None, recoverable: bool = False) -> None:
+        super().__init__(message)
+        self.step = step
+        self.recoverable = recoverable
+
+
+def _is_fastembed_model_cached() -> bool:
+    """Return True if the ONNX model already exists in ~/.contextual/models/."""
+    onnx_path = _find_onnx_model_path(MODELS_DIR)
+    return onnx_path is not None and onnx_path.exists()
+
+
+def _verify_setup() -> None:
+    from contextual import __version__ as current_version
+
+    if not VERSION_FILE.exists():
+        print_error(
+            "Global setup not found.",
+            hint="Run: contextual setup",
+        )
+        raise typer.Exit(1)
+
+    if not GLOBAL_CONFIG.exists():
+        print_error(
+            "Global config missing.",
+            hint="Run: contextual setup",
+        )
+        raise typer.Exit(1)
+
+    installed = VERSION_FILE.read_text(encoding="utf-8").strip()
+    if installed != current_version:
+        print_error(
+            f"Version mismatch: installed={installed}, current={current_version}.",
+            hint="Run: contextual setup to upgrade.",
+        )
+        raise typer.Exit(1)
+
+
+async def download_embed_model() -> None:
+    """Download the fastembed ONNX model into ~/.contextual/models/ and pin its hash."""
+    logger = structlog.get_logger()
+
+    with model_download_progress(_FASTEMBED_MODEL, size_hint="~130MB"):
+        try:
+            def _load() -> None:
+                from fastembed import TextEmbedding
+                MODELS_DIR.mkdir(parents=True, exist_ok=True)
+                # cache_dir routes fastembed to store weights under ~/.contextual/models/
+                # using HuggingFace Hub directory layout.
+                model = TextEmbedding(
+                    model_name=_FASTEMBED_MODEL,
+                    threads=1,
+                    cache_dir=str(MODELS_DIR),
+                )
+                # One inference pass — verifies the ONNX session loads correctly.
+                list(model.embed(["warmup"], batch_size=1))
+
+            await asyncio.to_thread(_load)
+        except Exception as exc:
+            raise FetchError(
+                f"Failed to download fastembed model: {exc}", step="embed"
+            ) from exc
+
+    # Pin the BLAKE3 hash of the ONNX file so _load_model() can verify integrity
+    # on every subsequent load without re-downloading.
+    try:
+        import blake3 as _blake3
+
+        onnx_path = _find_onnx_model_path(MODELS_DIR)
+        if onnx_path and onnx_path.exists():
+            pin = _blake3.blake3(onnx_path.read_bytes()).hexdigest()
+            ONNX_PIN_FILE.write_text(pin, encoding="utf-8")
+            logger.info(
+                "onnx_pin_written",
+                hash_prefix=pin[:16],
+                onnx_path=str(onnx_path),
+            )
+        else:
+            logger.warning("onnx_pin_skipped", reason="model_file_not_found_after_download")
+    except Exception as exc:
+        logger.warning("onnx_pin_write_failed", error=str(exc))
+
+    logger.info("fastembed_model_ready", model=_FASTEMBED_MODEL, cache_dir=str(MODELS_DIR))
+
+
+@register_command()
+@require_config
+def fetch(
+    embed: bool = typer.Option(False, "--embed", help="Download the Nomic embed model (ONNX)."),
+    all_: bool = typer.Option(False, "--all", help="Download all models (default)."),
+) -> None:
+    """Pre-download fastembed ONNX model weights into ~/.contextual/models/.
+
+    Model is stored in the global Contextual directory alongside the LanceDB
+    index and daemon config. Running this before the first `contextual index`
+    avoids an inline download during indexing and pins the BLAKE3 integrity hash.
+    """
+    _verify_setup()
+
+    print_fetch_start([_EMBED_FRIENDLY])
+
+    downloaded: list[str] = []
+    already_present: list[str] = []
+
+    if _is_fastembed_model_cached():
+        already_present.append(_EMBED_FRIENDLY)
+        # Re-pin even if cached — updates stale or missing pin file.
+        try:
+            import blake3 as _blake3
+
+            onnx_path = _find_onnx_model_path(MODELS_DIR)
+            if onnx_path and onnx_path.exists():
+                pin = _blake3.blake3(onnx_path.read_bytes()).hexdigest()
+                ONNX_PIN_FILE.write_text(pin, encoding="utf-8")
+        except Exception:
+            pass
+    else:
+        try:
+            asyncio.run(download_embed_model())
+            downloaded.append(_EMBED_FRIENDLY)
+        except FetchError as exc:
+            print_error(str(exc), hint="contextual fetch --embed")
+            raise typer.Exit(1) from exc
+
+    print_fetch_complete(downloaded, already_present)

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/cli/commands/index.py

@@ -47,6 +47,15 @@ async def index(
         "-f",
         help="Force full re-index even if already up-to-date",
     ),
+    bootstrap_cochange: bool = typer.Option(
+        False,
+        "--bootstrap-cochange",
+        help=(
+            "Walk up to 2000 git commits to pre-populate entity_commits for "
+            "co-change analysis. Only effective with --force. Commits touching "
+            ">50 files are excluded as noise."
+        ),
+    ),
     ctx: str | None = typer.Option(None, hidden=True),
 ) -> None:
     """Index codebase for semantic search."""
@@ -86,16 +95,11 @@ async def index(
         try:
             with index_progress() as prog:
                 try:
-                    # Suppress BLAKE3 pin warning during model load — pins are not
-                    # registered until `contextual fetch` is run with a real model.
-                    # The warning fires during SentenceTransformer.__init__, before
-                    # the outer warnings.simplefilter("ignore") wrapper takes effect.
                     warnings.filterwarnings(
                         "ignore",
                         message=".*No BLAKE3 hash pins.*",
                         category=UserWarning,
                     )
-                    # Reranker is search-only — never load at index time (~1.2GB saved).
                     await ensure_models_warm(models=["code", "docs"])
 
                     chunking_started = False
@@ -151,6 +155,7 @@ async def index(
                                             context=cli_ctx.request_context,
                                             force=force,
                                             incremental=incremental,
+                                            bootstrap_cochange=bootstrap_cochange,
                                             on_discovery=_on_discovery,
                                             on_file_chunked=_on_file_chunked,
                                             on_batch_embedded=_on_batch_embedded,

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/cli/commands/initialisation.py

@@ -531,6 +531,28 @@ async def run_workspace_init(workspace_root: Path) -> None:
 
         await _register_workspace_globally(global_conn, workspace_id, workspace_root, logger)
 
+        # Also register in workspaces.json for multi-workspace MCP support.
+        # Non-fatal: the DB registration above is authoritative for init; the
+        # registry entry is a convenience that `contextual workspace add` can
+        # replicate if this fails.
+        try:
+            from contextual.mcp.workspace_registry import WORKSPACE_REGISTRY
+
+            try:
+                WORKSPACE_REGISTRY.add_workspace(workspace_root.name, workspace_root)
+                logger.info(
+                    "workspace_registry_json_added",
+                    name=workspace_root.name,
+                    path=str(workspace_root),
+                )
+            except ValueError:
+                logger.debug(
+                    "workspace_registry_json_already_registered",
+                    name=workspace_root.name,
+                )
+        except Exception as exc:
+            logger.warning("workspace_registry_json_add_failed", error=str(exc))
+
         try:
             if not (
                 config_path.exists()

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/cli/commands/install.py

@@ -103,8 +103,7 @@ def _write_default_config() -> None:
         'tier = "free"\n\n'
         "[models]\n"
         'embed_model = "nomic-ai/nomic-embed-text-v1.5"\n'
-        'rerank_model = "jinaai/jina-reranker-v2-base-multilingual"\n'
-        f'models_dir = "{MODELS_DIR}"\n'
+f'models_dir = "{MODELS_DIR}"\n'
         f'cache_dir = "{MODELS_CACHE_DIR}"\n\n'
         "[storage]\n"
         f'global_db_path = "{GLOBAL_DB_DIR}"\n'

contextual_engine-0.9.6/contextual/cli/commands/workspace.py

@@ -0,0 +1,100 @@
+"""Workspace management commands."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+from contextual.cli.core import register_command
+from contextual.cli.core.output import print_error, print_workspace_list
+
+
+@register_command(group="workspace", name="list")
+async def workspace_list() -> None:
+    """List all registered workspaces."""
+    from contextual.mcp.workspace_registry import WORKSPACE_REGISTRY
+
+    ws_map = WORKSPACE_REGISTRY.load_registered_workspaces()
+    if not ws_map:
+        print_workspace_list([])
+        return
+
+    workspaces = []
+    for name, path_str in ws_map.items():
+        path = Path(path_str)
+        workspace_id = ""
+        last_indexed_at = "never"
+
+        try:
+            state_file = path / ".contextual" / "state.json"
+            if state_file.exists():
+                state_data = json.loads(state_file.read_text(encoding="utf-8"))
+                last_indexed_at = state_data.get("last_indexed_at", "never")
+        except Exception:
+            pass
+
+        try:
+            from contextual.indexing._config import load_runtime_config
+
+            cfg = load_runtime_config(path)
+            workspace_id = str(cfg.workspace_id)
+        except Exception:
+            pass
+
+        workspaces.append({
+            "path": f"{name}  {path_str}",
+            "workspace_id": workspace_id,
+            "last_indexed_at": last_indexed_at,
+        })
+
+    print_workspace_list(workspaces)
+
+
+@register_command(group="workspace", name="add")
+async def workspace_add(
+    path: str = typer.Argument(..., help="Absolute path to the workspace root"),
+    name: str = typer.Option(
+        None,
+        "--name",
+        "-n",
+        help="Human-readable workspace name (defaults to directory basename)",
+    ),
+) -> None:
+    """Register a workspace in the multi-workspace registry."""
+    from contextual.mcp.workspace_registry import WORKSPACE_REGISTRY
+
+    resolved = Path(path).expanduser().resolve()
+    ws_name = name or resolved.name
+    if not ws_name:
+        print_error("Could not derive a workspace name.", hint="Use --name <name>")
+        raise typer.Exit(1)
+
+    try:
+        WORKSPACE_REGISTRY.add_workspace(ws_name, resolved)
+        typer.echo(f"Registered workspace '{ws_name}' at {resolved}")
+    except ValueError as exc:
+        print_error(str(exc))
+        raise typer.Exit(1) from exc
+    except Exception as exc:
+        print_error(f"Failed to register workspace: {exc}")
+        raise typer.Exit(1) from exc
+
+
+@register_command(group="workspace", name="remove")
+async def workspace_remove(
+    name: str = typer.Argument(..., help="Workspace name to deregister"),
+) -> None:
+    """Remove a workspace from the multi-workspace registry."""
+    from contextual.mcp.workspace_registry import WORKSPACE_REGISTRY
+
+    try:
+        WORKSPACE_REGISTRY.remove_workspace(name)
+        typer.echo(f"Removed workspace '{name}'")
+    except KeyError as exc:
+        print_error(str(exc))
+        raise typer.Exit(1) from exc
+    except Exception as exc:
+        print_error(f"Failed to remove workspace: {exc}")
+        raise typer.Exit(1) from exc

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/config/__init__.py

@@ -19,7 +19,6 @@ from contextual.config.workspace.retrieval_config import (
     BM25Config,
     DenseConfig,
     MMRConfig,
-    RerankerConfig,
     RetrievalConfig,
 )
 from contextual.config.workspace.workspace_config import WorkspaceConfig
@@ -109,7 +108,6 @@ __all__ = [
     "MMRConfig",
     "ObservabilityConfig",
     "PipelineConfig",
-    "RerankerConfig",
     "RetrievalConfig",
     "SecurityConfig",
     "StorageConfig",

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/config/base.py

@@ -25,7 +25,7 @@ class ContextualConfig(BaseModel):
         project_root: Absolute path to the project being indexed.
         indexing: Indexing configuration for code and workspace files.
         embedding: Embedding model configuration.
-        retrieval: Retrieval configuration for search and reranking.
+        retrieval: Retrieval configuration for search (RRF fusion + MMR).
         storage: Storage configuration for database and index paths.
         security: Security configuration for filesystem and workspace policy.
     """

{contextual_engine-0.9.4 → contextual_engine-0.9.6}/contextual/config/global_/mcp_config.py

@@ -34,10 +34,16 @@ class DaemonConfig(BaseModel):
     port: int = 9091                   # must match _platform.WINDOWS_PORT/TCP_PORT
 
     # Lifecycle
-    idle_sleep_minutes: int = 30       # Minutes idle before daemon sleeps (releases RAM)
-    die_after_sleep_hours: float = 2.0 # Hours asleep before daemon fully exits (0 = never)
-    startup_timeout: float = 45.0      # Seconds to wait for the daemon to start serving
-                                       # (model load is 20-50s on 8GB unified memory)
+    # idle_sleep_minutes: minutes of no incoming MCP requests before the daemon
+    # unloads the ONNX model and enters sleep state. Active shim connections
+    # (idle SSE sessions) do NOT prevent this — only actual HTTP requests reset
+    # the clock.  die_after_sleep_hours: hours asleep before a clean SIGTERM
+    # exit. Total inactivity before full exit = idle_sleep_minutes +
+    # die_after_sleep_hours * 60.  Set die_after_sleep_hours = 0 to never exit.
+    idle_sleep_minutes: int = 30       # Minutes of no requests before sleeping
+    die_after_sleep_hours: float = 1.0 # Hours asleep before clean exit (0 = never)
+    startup_timeout: float = 15.0      # Seconds to wait for the daemon to start serving
+                                       # (ONNX model loads in ~2-5s from ~/.contextual/models/)
 
     # Rate limiting
     rate_limit_per_minute: int = 120   # Max tool calls per minute per client
@@ -53,11 +59,13 @@ class DaemonConfig(BaseModel):
     # Heap / memory trimmer
     # When the daemon's physical footprint exceeds heap_trim_threshold_mb a
     # hard trim fires (GC + malloc page return). If the footprint is still over
-    # the threshold on the next tick and no sessions are active, models are
-    # offloaded to CPU to recover MPS/GPU pages. Set to 0 to disable.
+    # the threshold on the next tick and no sessions are active, the ONNX model
+    # session is unloaded (freed from memory). Set to 0 to disable.
+    # 1500 MB accounts for: ONNX session (~130 MB) + LanceDB PyArrow buffers
+    # (~400-800 MB active) + graph/cache overhead — triggers before OOM pressure.
     heap_monitor_enabled: bool = True
     heap_monitor_interval_seconds: int = 30
-    heap_trim_threshold_mb: int = 2500
+    heap_trim_threshold_mb: int = 1500
 
     # Logging
     log_level: str = "INFO"