contextual-engine 0.1.0__py3-none-any.whl

contextual/security/workspace.py

@@ -0,0 +1,348 @@
+"""Workspace isolation and management.
+
+This module provides per-project workspace isolation. Each project gets its own
+.contextual/ directory containing:
+- SQLite database (contextual.db)
+- LanceDB vector store (lance/)
+- Tantivy BM25 index (tantivy/)
+- Configuration (config.toml)
+- Logs (logs/)
+
+Workspace isolation prevents:
+- Cross-project data leakage
+- Permission escalation
+- Global state contamination
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from contextual.core.errors import ErrorCode, SecurityError, security_context
+from contextual.security.paths import safe_path
+
+
+class WorkspaceManager:
+    """Manager for per-project workspaces.
+    
+    Each project gets an isolated .contextual/ directory with proper permissions.
+    This class handles workspace initialization, validation, and path resolution.
+    
+    Attributes:
+        project_root: Absolute path to project being indexed.
+        workspace_dir: Path to .contextual/ directory.
+    """
+
+    def __init__(self, project_root: Path) -> None:
+        """Initialize workspace manager.
+        
+        Args:
+            project_root: Absolute path to project root.
+            
+        Raises:
+            SecurityError: If project_root is invalid.
+        """
+        if not project_root.is_absolute():
+            msg = "Project root must be absolute"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_INIT_FAILED,
+                message=msg,
+                context=security_context(path=str(project_root)),
+            )
+
+        if not project_root.exists():
+            msg = "Project root does not exist"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_INIT_FAILED,
+                message=msg,
+                context=security_context(path=str(project_root)),
+            )
+
+        if not project_root.is_dir():
+            msg = "Project root is not a directory"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_INIT_FAILED,
+                message=msg,
+                context=security_context(path=str(project_root)),
+            )
+
+        self.project_root = project_root
+        self.workspace_dir = project_root / ".contextual"
+
+    def init_workspace(
+        self,
+        *,
+        workspace_permissions: int = 0o700,
+        db_permissions: int = 0o600,
+    ) -> None:
+        """Initialize workspace directory with proper permissions.
+        
+        Creates .contextual/ directory and subdirectories if they don't exist.
+        Sets strict permissions to prevent unauthorized access.
+        
+        Directory structure:
+        .contextual/
+        ├── contextual.db      (SQLite database)
+        ├── lance/             (LanceDB vector store)
+        ├── tantivy/           (Tantivy BM25 index)
+        ├── config.toml        (Project-local config)
+        └── logs/              (Log files)
+        
+        Args:
+            workspace_permissions: Permissions for .contextual/ directory (default 0o700).
+            db_permissions: Permissions for database files (default 0o600).
+            
+        Raises:
+            SecurityError: If workspace initialization fails.
+        """
+        # Create workspace directory
+        try:
+            self.workspace_dir.mkdir(mode=workspace_permissions, exist_ok=True)
+        except OSError as e:
+            msg = f"Failed to create workspace directory: {e}"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_INIT_FAILED,
+                message=msg,
+                context=security_context(path=str(self.workspace_dir)),
+            ) from e
+
+        # Set permissions explicitly (mkdir mode can be affected by umask)
+        try:
+            os.chmod(self.workspace_dir, workspace_permissions)
+        except OSError as e:
+            msg = f"Failed to set workspace permissions: {e}"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_PERMISSION_DENIED,
+                message=msg,
+                context=security_context(path=str(self.workspace_dir)),
+            ) from e
+
+        # Create subdirectories
+        subdirs = ["lance", "tantivy", "logs"]
+        for subdir in subdirs:
+            subdir_path = self.workspace_dir / subdir
+            try:
+                subdir_path.mkdir(mode=workspace_permissions, exist_ok=True)
+                os.chmod(subdir_path, workspace_permissions)
+            except OSError as e:
+                msg = f"Failed to create subdirectory {subdir}: {e}"
+                raise SecurityError(
+                    code=ErrorCode.SECURITY_WORKSPACE_INIT_FAILED,
+                    message=msg,
+                    context=security_context(path=str(subdir_path)),
+                ) from e
+
+        # Create .gitignore to exclude workspace from version control
+        gitignore_path = self.workspace_dir / ".gitignore"
+        if not gitignore_path.exists():
+            gitignore_content = (
+                "# Contextual workspace - do not commit\n"
+                "*\n"
+                "!.gitignore\n"
+                "!config.toml\n"
+            )
+            try:
+                gitignore_path.write_text(gitignore_content)
+                os.chmod(gitignore_path, 0o644)
+            except OSError:
+                # Non-fatal - workspace can function without .gitignore
+                pass
+
+    def get_db_path(self) -> Path:
+        """Get path to SQLite database file.
+        
+        Returns:
+            Path to contextual.db within workspace.
+            
+        Raises:
+            SecurityError: If path validation fails.
+        """
+        return self._get_workspace_path("contextual.db")
+
+    def get_lance_dir(self) -> Path:
+        """Get path to LanceDB directory.
+        
+        Returns:
+            Path to lance/ subdirectory within workspace.
+            
+        Raises:
+            SecurityError: If path validation fails.
+        """
+        return self._get_workspace_path("lance")
+
+    def get_tantivy_dir(self) -> Path:
+        """Get path to Tantivy index directory.
+        
+        Returns:
+            Path to tantivy/ subdirectory within workspace.
+            
+        Raises:
+            SecurityError: If path validation fails.
+        """
+        return self._get_workspace_path("tantivy")
+
+    def get_config_path(self) -> Path:
+        """Get path to project-local config file.
+        
+        Returns:
+            Path to config.toml within workspace.
+            
+        Raises:
+            SecurityError: If path validation fails.
+        """
+        return self._get_workspace_path("config.toml")
+
+    def get_log_dir(self) -> Path:
+        """Get path to log directory.
+        
+        Returns:
+            Path to logs/ subdirectory within workspace.
+            
+        Raises:
+            SecurityError: If path validation fails.
+        """
+        return self._get_workspace_path("logs")
+
+    def _get_workspace_path(self, relative_path: str) -> Path:
+        """Get a path within the workspace with safety validation.
+        
+        Args:
+            relative_path: Path relative to workspace directory.
+            
+        Returns:
+            Validated absolute path within workspace.
+            
+        Raises:
+            SecurityError: If path is invalid or outside workspace.
+        """
+        # Use safe_path to validate (allow_create for files that might not exist yet)
+        return safe_path(self.workspace_dir, relative_path, allow_create=True)
+
+    def ensure_workspace_exists(self) -> None:
+        """Ensure workspace directory exists.
+        
+        Creates workspace if it doesn't exist.
+        
+        Raises:
+            SecurityError: If workspace initialization fails.
+        """
+        if not self.workspace_dir.exists():
+            self.init_workspace()
+
+    def validate_workspace_isolation(self) -> None:
+        """Validate that workspace is properly isolated.
+        
+        Checks:
+        1. Workspace directory exists within project root
+        2. Workspace directory has correct permissions
+        3. No symlinks pointing outside project root
+        
+        Raises:
+            SecurityError: If isolation is violated.
+        """
+        # Check workspace exists
+        if not self.workspace_dir.exists():
+            msg = "Workspace directory does not exist"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_ISOLATION_VIOLATION,
+                message=msg,
+                context=security_context(path=str(self.workspace_dir)),
+            )
+
+        # Check workspace is within project root
+        try:
+            self.workspace_dir.relative_to(self.project_root)
+        except ValueError as e:
+            msg = "Workspace directory is outside project root"
+            raise SecurityError(
+                code=ErrorCode.SECURITY_WORKSPACE_ISOLATION_VIOLATION,
+                message=msg,
+                context=security_context(
+                    path=str(self.workspace_dir),
+                    root=str(self.project_root),
+                ),
+            ) from e
+
+        # Check workspace is not a symlink pointing outside project root
+        if self.workspace_dir.is_symlink():
+            try:
+                resolved = self.workspace_dir.resolve(strict=True)
+                resolved.relative_to(self.project_root)
+            except (ValueError, OSError, RuntimeError) as e:
+                msg = "Workspace symlink points outside project root"
+                raise SecurityError(
+                    code=ErrorCode.SECURITY_WORKSPACE_ISOLATION_VIOLATION,
+                    message=msg,
+                    context=security_context(path=str(self.workspace_dir)),
+                ) from e
+
+        # Check permissions (on Unix-like systems)
+        if hasattr(os, "stat"):
+            try:
+                stat_info = self.workspace_dir.stat()
+                mode = stat_info.st_mode & 0o777
+
+                # Warn if permissions are too permissive (group/other have access)
+                if mode & 0o077:  # Group or other have any permissions
+                    # This is a warning, not an error - user might have reasons
+                    # for broader permissions in a trusted environment
+                    pass
+            except OSError:
+                # Permission check failed - non-fatal
+                pass
+
+    def get_workspace_info(self) -> dict[str, str]:
+        """Get information about workspace.
+        
+        Returns:
+            Dictionary with workspace paths and status.
+        """
+        return {
+            "project_root": str(self.project_root),
+            "workspace_dir": str(self.workspace_dir),
+            "db_path": str(self.get_db_path()),
+            "lance_dir": str(self.get_lance_dir()),
+            "tantivy_dir": str(self.get_tantivy_dir()),
+            "config_path": str(self.get_config_path()),
+            "log_dir": str(self.get_log_dir()),
+            "exists": str(self.workspace_dir.exists()),
+        }
+
+
+def init_workspace(project_root: Path) -> WorkspaceManager:
+    """Initialize a workspace for a project.
+    
+    Convenience function that creates a WorkspaceManager and initializes
+    the workspace directory.
+    
+    Args:
+        project_root: Absolute path to project root.
+        
+    Returns:
+        Initialized WorkspaceManager.
+        
+    Raises:
+        SecurityError: If initialization fails.
+    """
+    manager = WorkspaceManager(project_root)
+    manager.init_workspace()
+    return manager
+
+
+def get_workspace_manager(project_root: Path) -> WorkspaceManager:
+    """Get a workspace manager for a project.
+    
+    Creates WorkspaceManager but does not initialize workspace directory.
+    Use this when you want to check workspace status before initialization.
+    
+    Args:
+        project_root: Absolute path to project root.
+        
+    Returns:
+        WorkspaceManager instance.
+        
+    Raises:
+        SecurityError: If project_root is invalid.
+    """
+    return WorkspaceManager(project_root)

contextual/storage/__init__.py

@@ -0,0 +1,36 @@
+"""Storage layer: SQLite, LanceDB, and bi-temporal query infrastructure.
+
+Provides connection pooling, schema management, migrations, and temporal query helpers.
+All persistence operations go through this layer.
+"""
+from __future__ import annotations
+
+from contextual.storage.migrations import (
+    apply_initial_schema,
+    get_current_version,
+    record_migration,
+    run_migrations,
+)
+from contextual.storage.schema import apply_schema
+from contextual.storage.sqlite_pool import SQLitePool
+from contextual.storage.vec0_manager import (
+    count_vectors,
+    delete_vectors,
+    insert_vectors,
+    search_vectors,
+)
+
+
+__all__ = [
+    "SQLitePool",
+    "apply_initial_schema",
+    "apply_schema",
+    "count_vectors",
+    "delete_vectors",
+    "get_current_version",
+    "insert_vectors",
+    "record_migration",
+    "run_migrations",
+    "search_vectors",
+]
+

contextual/storage/fts_manager.py

@@ -0,0 +1,273 @@
+"""FTS5 full-text search manager for code chunks."""
+
+from __future__ import annotations
+
+import sqlite3
+from typing import TYPE_CHECKING
+
+from contextual.core.errors import ErrorCode, StorageError, storage_context
+from contextual.observability.logging import get_logger
+
+
+if TYPE_CHECKING:
+    from sqlite3 import Connection
+
+logger = get_logger(__name__)
+
+_FTS_TABLE = "fts_chunks"
+_FTS_ERROR_CODE = getattr(ErrorCode, "STORAGE_FTS_FAILED", ErrorCode.STORAGE_SQLITE_QUERY_FAILED)
+
+
+def escape_fts_query(query: str) -> str:
+    """Escape a user query for safe FTS5 phrase matching.
+
+    Args:
+        query: Raw user search text.
+
+    Returns:
+        Phrase-quoted FTS5-safe query string.
+    """
+    escaped = query.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def search_fts(
+    conn: Connection,
+    query: str,
+    k: int = 10,
+    language_filter: str | None = None,
+) -> list[tuple[int, float]]:
+    """Search chunks using SQLite FTS5 and return ranked BM25 scores.
+
+    Args:
+        conn: SQLite connection.
+        query: FTS5 query text.
+        k: Maximum result count in range [1, 1000].
+        language_filter: Optional language suffix (currently ignored - can be implemented via joins).
+
+    Returns:
+        List of `(chunk_id, score)` tuples where score is positive and higher is better.
+
+    Raises:
+        StorageError: If SQLite execution fails unexpectedly.
+    """
+    if query is None:
+        return []
+    if not isinstance(query, str) or not query.strip():
+        return []
+    if k <= 0 or k > 1000:
+        raise ValueError("k must be between 1 and 1000")
+
+    normalized_query = query.strip()
+    escaped_query = escape_fts_query(normalized_query)
+
+    sql = (
+        "SELECT chunk_id, -bm25(fts_chunks) AS score "
+        "FROM fts_chunks "
+        "WHERE fts_chunks MATCH ?"
+    )
+    params: list[object] = [escaped_query]
+
+    # Language filtering can be added later via JOIN with files table if needed
+    # TODO: Implement language filtering via JOIN with files table
+
+    sql += " ORDER BY score DESC LIMIT ?"
+    params.append(k)
+
+    try:
+        cursor = conn.execute(sql, tuple(params))
+        rows = cursor.fetchall()
+        return [(int(chunk_id), float(score)) for chunk_id, score in rows]
+    except sqlite3.OperationalError as exc:
+        # Invalid FTS syntax should not break callers; return no results.
+        logger.warning(
+            "Invalid FTS5 query; returning empty results",
+            query=normalized_query,
+            error=str(exc),
+        )
+        return []
+    except sqlite3.Error as exc:
+        msg = f"FTS5 search failed: {exc}"
+        logger.error("FTS5 search failed", error=str(exc))
+        raise StorageError(
+            code=_FTS_ERROR_CODE,
+            message=msg,
+            context=storage_context(table=_FTS_TABLE, query=normalized_query),
+        ) from exc
+
+
+def count_matches(conn: Connection, query: str) -> int:
+    """Count total chunks that match an FTS5 query.
+
+    Args:
+        conn: SQLite connection.
+        query: FTS5 query text.
+
+    Returns:
+        Number of matching chunks.
+
+    Raises:
+        StorageError: If SQLite execution fails unexpectedly.
+    """
+    if query is None:
+        return 0
+    if not isinstance(query, str) or not query.strip():
+        return 0
+
+    normalized_query = query.strip()
+    escaped_query = escape_fts_query(normalized_query)
+
+    sql = (
+        "SELECT COUNT(*) "
+        "FROM fts_chunks "
+        "WHERE fts_chunks MATCH ?"
+    )
+
+    try:
+        cursor = conn.execute(sql, (escaped_query,))
+        row = cursor.fetchone()
+        return int(row[0]) if row else 0
+    except sqlite3.OperationalError as exc:
+        logger.warning(
+            "Invalid FTS5 count query; returning zero",
+            query=normalized_query,
+            error=str(exc),
+        )
+        return 0
+    except sqlite3.Error as exc:
+        msg = f"FTS5 count failed: {exc}"
+        logger.error("FTS5 count failed", error=str(exc))
+        raise StorageError(
+            code=_FTS_ERROR_CODE,
+            message=msg,
+            context=storage_context(table=_FTS_TABLE, query=normalized_query),
+        ) from exc
+
+
+def get_chunk_snippet(
+    conn: Connection,
+    chunk_id: int,
+    query: str,
+    max_len: int = 200,
+) -> str:
+    """Get a highlighted snippet for a matching chunk.
+
+    Args:
+        conn: SQLite connection.
+        chunk_id: Chunk identifier from `chunks.id`.
+        query: Query text used for highlighting.
+        max_len: Maximum snippet character length.
+
+    Returns:
+        Highlighted snippet string, truncated to `max_len`, or empty string.
+
+    Raises:
+        StorageError: If SQLite execution fails unexpectedly.
+    """
+    if query is None:
+        return ""
+    if not isinstance(query, str) or not query.strip():
+        return ""
+    if max_len <= 0:
+        return ""
+
+    normalized_query = query.strip()
+    escaped_query = escape_fts_query(normalized_query)
+
+    sql = (
+        "SELECT snippet(fts_chunks, 1, '<b>', '</b>', '...', 20) AS snippet_text "
+        "FROM fts_chunks "
+        "WHERE chunk_id = ? AND fts_chunks MATCH ?"
+    )
+
+    try:
+        cursor = conn.execute(sql, (chunk_id, escaped_query))
+        row = cursor.fetchone()
+        if not row or row[0] is None:
+            return ""
+        snippet = str(row[0])
+        return snippet if len(snippet) <= max_len else snippet[:max_len]
+    except sqlite3.OperationalError as exc:
+        logger.warning(
+            "Invalid FTS5 snippet query; returning empty snippet",
+            query=normalized_query,
+            chunk_id=chunk_id,
+            error=str(exc),
+        )
+        return ""
+    except sqlite3.Error as exc:
+        msg = f"FTS5 snippet retrieval failed: {exc}"
+        logger.error("FTS5 snippet retrieval failed", error=str(exc), chunk_id=chunk_id)
+        raise StorageError(
+            code=_FTS_ERROR_CODE,
+            message=msg,
+            context=storage_context(table=_FTS_TABLE, query=normalized_query),
+        ) from exc
+
+
+def test_fts_search() -> None:
+    """Validate core FTS5 search and escaping behavior."""
+    conn = sqlite3.connect(":memory:")
+
+    conn.execute("""
+        CREATE TABLE chunks (
+            id INTEGER PRIMARY KEY,
+            file_id INTEGER,
+            start_line INTEGER,
+            end_line INTEGER,
+            symbol_name TEXT,
+            symbol_type TEXT,
+            content TEXT,
+            content_hash TEXT,
+            embedding_id INTEGER,
+            indexed_at INTEGER
+        )
+    """)
+
+    conn.execute("""
+        CREATE VIRTUAL TABLE fts_chunks USING fts5(
+            chunk_id UNINDEXED,
+            content,
+            symbol_name,
+            tokenize='unicode61 remove_diacritics 2'
+        )
+    """)
+
+    conn.execute(
+        "INSERT INTO chunks VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+        (1, 1, 10, 20, "getUserData", "function", "def getUserData(): return data", "hash1", None, 1000),
+    )
+    conn.execute(
+        "INSERT INTO fts_chunks VALUES (?, ?, ?)",
+        (1, "def getUserData(): return data", "getUserData"),
+    )
+
+    results = search_fts(conn, "getUserData", k=10)
+    assert len(results) == 1
+    assert results[0][0] == 1
+    assert results[0][1] > 0
+
+    escaped = escape_fts_query('test "quote"')
+    assert '"' in escaped
+
+
+def test_fts_helpers() -> None:
+    """Validate count and snippet helper functions."""
+    conn = sqlite3.connect(":memory:")
+    conn.execute("""
+        CREATE VIRTUAL TABLE fts_chunks USING fts5(
+            chunk_id UNINDEXED,
+            content,
+            symbol_name,
+            tokenize='unicode61 remove_diacritics 2'
+        )
+    """)
+    conn.execute(
+        "INSERT INTO fts_chunks VALUES (?, ?, ?)",
+        (11, "class Service: async def run(self): pass", "Service"),
+    )
+
+    assert count_matches(conn, "Service") == 1
+    snippet = get_chunk_snippet(conn, chunk_id=11, query="async def")
+    assert snippet
+    assert isinstance(snippet, str)