ifcraftcorpus 1.2.1__tar.gz → 1.4.0__tar.gz

{ifcraftcorpus-1.2.1 → ifcraftcorpus-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ifcraftcorpus
-Version: 1.2.1
+Version: 1.4.0
 Summary: Interactive fiction craft corpus with search library and MCP server
 Project-URL: Homepage, https://pvliesdonk.github.io/if-craft-corpus
 Project-URL: Repository, https://github.com/pvliesdonk/if-craft-corpus

{ifcraftcorpus-1.2.1 → ifcraftcorpus-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ifcraftcorpus"
-version = "1.2.1"
+version = "1.4.0"
 description = "Interactive fiction craft corpus with search library and MCP server"
 readme = "README.md"
 license = {text = "MIT"}

{ifcraftcorpus-1.2.1 → ifcraftcorpus-1.4.0}/src/ifcraftcorpus/cli.py

@@ -125,7 +125,10 @@ def cmd_embeddings_status(args: argparse.Namespace) -> int:
         status = "✓ Available" if available else "✗ Not available"
         print(f"{name:20} {status}")
         if available:
-            print(f"{'':20} Model: {provider.model} ({provider.dimension}d)")
+            extra_info = ""
+            if hasattr(provider, "cpu_only") and provider.cpu_only:
+                extra_info = " [CPU-only]"
+            print(f"{'':20} Model: {provider.model} ({provider.dimension}d){extra_info}")
 
     # Auto-detect
     print("\n=== Auto-Detection ===\n")
@@ -164,9 +167,10 @@ def cmd_embeddings_build(args: argparse.Namespace) -> int:
 
     # Get provider
     provider: EmbeddingProvider | None = None
+    cpu_only = getattr(args, "cpu_only", False)
     if args.provider:
         if args.provider == "ollama":
-            provider = OllamaEmbeddings(model=args.model, host=args.ollama_host)
+            provider = OllamaEmbeddings(model=args.model, host=args.ollama_host, cpu_only=cpu_only)
         elif args.provider == "openai":
             provider = OpenAIEmbeddings(model=args.model, api_key=args.openai_key)
         elif args.provider in ("sentence-transformers", "st", "local"):
@@ -175,7 +179,7 @@ def cmd_embeddings_build(args: argparse.Namespace) -> int:
             print(f"Unknown provider: {args.provider}", file=sys.stderr)
             return 1
     else:
-        provider = get_embedding_provider(model=args.model)
+        provider = get_embedding_provider(model=args.model, cpu_only=cpu_only)
 
     if not provider:
         print("No embedding provider available.", file=sys.stderr)
@@ -186,13 +190,19 @@ def cmd_embeddings_build(args: argparse.Namespace) -> int:
         print(f"Provider {provider.provider_name} is not available.", file=sys.stderr)
         return 1
 
+    # Show CPU-only status for Ollama
+    cpu_only_status = ""
+    if hasattr(provider, "cpu_only") and provider.cpu_only:
+        cpu_only_status = " (CPU-only)"
+
     logger.info(
-        "CLI embeddings build provider=%s model=%s output=%s",
+        "CLI embeddings build provider=%s model=%s output=%s cpu_only=%s",
         provider.provider_name,
         provider.model,
         args.output,
+        getattr(provider, "cpu_only", False),
     )
-    print(f"Using provider: {provider.provider_name}")
+    print(f"Using provider: {provider.provider_name}{cpu_only_status}")
     print(f"Model: {provider.model} ({provider.dimension}d)")
 
     # Build embeddings
@@ -308,6 +318,12 @@ def main() -> int:
     )
     emb_build.add_argument("--ollama-host", help="Ollama host URL")
     emb_build.add_argument("--openai-key", help="OpenAI API key")
+    emb_build.add_argument(
+        "--cpu-only",
+        action="store_true",
+        help="Force CPU-only inference for Ollama (num_gpu=0). "
+        "Useful when GPU is under VRAM pressure.",
+    )
     emb_build.set_defaults(func=cmd_embeddings_build)
 
     args = parser.parse_args()

{ifcraftcorpus-1.2.1 → ifcraftcorpus-1.4.0}/src/ifcraftcorpus/index.py

@@ -56,21 +56,58 @@ from ifcraftcorpus.parser import Document, parse_directory
 def _sanitize_fts_query(query: str) -> str:
     """Sanitize a query string for the FTS5 MATCH clause.
 
-    This function replaces hyphens with spaces to prevent FTS5 from
-    interpreting them as the `NOT` operator. This is intended to correctly
-    handle natural language queries with hyphenated words, for example
-    transforming "haunted-house" into a search for "haunted house".
+    This function replaces special characters that could cause FTS5 syntax
+    errors with spaces. This is intended to correctly handle natural language
+    queries from LLMs and users, for example transforming:
+    - "haunted-house" into "haunted house" (hyphen as NOT operator)
+    - "dialogue, subtext" into "dialogue subtext" (comma syntax error)
 
     It also collapses any resulting multiple spaces into a single space.
 
+    Note: This is used as a fallback when raw FTS5 queries fail. The search
+    method first tries the raw query to support advanced FTS5 syntax, then
+    falls back to sanitized query on syntax errors.
+    See https://github.com/pvliesdonk/if-craft-corpus/issues/10
+
     Args:
         query: Raw query string from user input.
 
     Returns:
         Sanitized query safe for FTS5 MATCH.
     """
-    # Replace hyphens and collapse whitespace in one go.
-    return " ".join(query.replace("-", " ").split())
+    # Replace problematic characters with spaces:
+    # - hyphen: FTS5 interprets as NOT operator
+    # - comma: FTS5 column list syntax
+    # - parentheses: FTS5 grouping syntax
+    # - curly braces: FTS5 column filter syntax
+    # - caret: FTS5 position marker
+    # - plus: FTS5 column weight
+    # - colon after words could affect column queries, but we preserve it
+    #   to allow intentional column:value syntax
+    # Using str.translate is more efficient for replacing multiple single characters.
+    translation_table = str.maketrans("-,(){}^+", " " * 8)
+    sanitized = query.translate(translation_table)
+    # Collapse whitespace
+    return " ".join(sanitized.split())
+
+
+def _is_fts5_query_error(error: sqlite3.OperationalError) -> bool:
+    """Check if an OperationalError is an FTS5 query parsing error.
+
+    Args:
+        error: The SQLite OperationalError to check.
+
+    Returns:
+        True if this is an FTS5 query error that might be recoverable
+        by sanitizing the query.
+    """
+    msg = str(error).lower()
+    # FTS5 syntax errors (e.g., "fts5: syntax error near ','")
+    if "fts5" in msg and "syntax error" in msg:
+        return True
+    # Column errors from FTS5 query parsing (e.g., "no such column: voice")
+    # This can happen when hyphens are interpreted as column filters
+    return "no such column" in msg
 
 
 @dataclass
@@ -359,51 +396,25 @@ class CorpusIndex:
             self.add_document(doc)
         return len(documents)
 
-    def search(
+    def _execute_fts_query(
         self,
-        query: str,
-        *,
-        cluster: str | None = None,
-        limit: int = 10,
+        fts_query: str,
+        cluster: str | None,
+        limit: int,
     ) -> list[SearchResult]:
-        """Search the corpus using FTS5 full-text search.
-
-        Performs a keyword search using SQLite FTS5 with BM25 ranking.
-        Supports the full FTS5 query syntax for advanced searches.
+        """Execute an FTS5 query and return results.
 
         Args:
-            query: Search query. Supports FTS5 syntax:
-
-                - Simple keywords: ``dialogue``
-                - Phrases: ``"character voice"``
-                - Boolean: ``tension AND suspense``, ``horror NOT comedy``
-                - Prefix: ``narrat*``
-                - Column-specific: ``title:craft``
-
-            cluster: Optional cluster name to filter results. Only returns
-                matches from the specified cluster.
-            limit: Maximum number of results to return. Default 10.
+            fts_query: The FTS5 query string.
+            cluster: Optional cluster filter.
+            limit: Maximum results to return.
 
         Returns:
-            List of :class:`SearchResult` objects, sorted by BM25 relevance
-            score in descending order (best matches first).
+            List of SearchResult objects.
 
-        Example:
-            >>> # Simple search
-            >>> results = index.search("dialogue")
-
-            >>> # Phrase search
-            >>> results = index.search('"character voice"')
-
-            >>> # Boolean with cluster filter
-            >>> results = index.search("tension OR suspense",
-            ...                        cluster="emotional-design",
-            ...                        limit=5)
+        Raises:
+            sqlite3.OperationalError: If the query has invalid FTS5 syntax.
         """
-        # Build FTS5 query - sanitize to handle special characters
-        fts_query = _sanitize_fts_query(query)
-
-        # Add cluster filter if specified
         where_clause = ""
         params: list[str | int] = [fts_query]
         if cluster:
@@ -447,6 +458,72 @@ class CorpusIndex:
 
         return results
 
+    def search(
+        self,
+        query: str,
+        *,
+        cluster: str | None = None,
+        limit: int = 10,
+    ) -> list[SearchResult]:
+        """Search the corpus using FTS5 full-text search.
+
+        Performs a keyword search using SQLite FTS5 with BM25 ranking.
+        Supports the full FTS5 query syntax for advanced searches.
+
+        The search first attempts to execute the query as-is to support
+        advanced FTS5 syntax. If that fails with a syntax error, it falls
+        back to a sanitized version of the query that treats special
+        characters as word separators.
+
+        Args:
+            query: Search query. Supports FTS5 syntax:
+
+                - Simple keywords: ``dialogue``
+                - Phrases: ``"character voice"``
+                - Boolean: ``tension AND suspense``, ``horror NOT comedy``
+                - Prefix: ``narrat*``
+                - Column-specific: ``title:craft``
+
+                Natural language queries with punctuation (e.g., "dialogue,
+                subtext") are also supported - they will be automatically
+                sanitized if they cause syntax errors.
+
+            cluster: Optional cluster name to filter results. Only returns
+                matches from the specified cluster.
+            limit: Maximum number of results to return. Default 10.
+
+        Returns:
+            List of :class:`SearchResult` objects, sorted by BM25 relevance
+            score in descending order (best matches first).
+
+        Example:
+            >>> # Simple search
+            >>> results = index.search("dialogue")
+
+            >>> # Phrase search
+            >>> results = index.search('"character voice"')
+
+            >>> # Boolean with cluster filter
+            >>> results = index.search("tension OR suspense",
+            ...                        cluster="emotional-design",
+            ...                        limit=5)
+
+            >>> # Natural language (auto-sanitized)
+            >>> results = index.search("dialogue, subtext")
+        """
+        # Try raw query first to support advanced FTS5 syntax
+        try:
+            return self._execute_fts_query(query, cluster, limit)
+        except sqlite3.OperationalError as e:
+            if not _is_fts5_query_error(e):
+                raise
+
+        # Fallback to sanitized query for natural language input
+        sanitized = _sanitize_fts_query(query)
+        if not sanitized:
+            return []
+        return self._execute_fts_query(sanitized, cluster, limit)
+
     def list_documents(self) -> list[dict[str, str]]:
         """List all indexed documents with their metadata.
 

{ifcraftcorpus-1.2.1 → ifcraftcorpus-1.4.0}/src/ifcraftcorpus/mcp_server.py

@@ -194,9 +194,24 @@ def search_corpus(
     worldbuilding, and genre conventions.
 
     Args:
-        query: Search query describing what craft guidance you need.
-               Examples: "dialogue subtext", "branching narrative",
-               "pacing action scenes", "horror atmosphere".
+        query: Search query. Supports natural language or FTS5 syntax:
+
+               Natural language examples:
+               - "dialogue subtext"
+               - "branching narrative"
+               - "pacing action scenes"
+
+               FTS5 advanced syntax:
+               - Exact phrases: '"character voice"'
+               - Boolean NOT: "dialogue NOT comedy"
+               - Boolean OR: "tension OR suspense"
+               - Boolean AND: "dialogue AND subtext"
+               - Prefix search: "narrat*"
+               - Column filter: "title:craft", "cluster:genre-conventions"
+
+               Natural language queries with punctuation are automatically
+               sanitized, so both styles work seamlessly.
+
         cluster: Optional topic cluster to filter by. Valid clusters:
                  narrative-structure, prose-and-language, genre-conventions,
                  audience-and-access, world-and-setting, emotional-design,
@@ -367,11 +382,15 @@ def embeddings_status() -> dict[str, Any]:
             try:
                 provider = cls()
                 available = provider.check_availability()
-                result["providers"][name] = {
+                provider_info: dict[str, Any] = {
                     "available": available,
                     "model": provider.model if available else None,
                     "dimension": provider.dimension if available else None,
                 }
+                # Add cpu_only info for Ollama
+                if hasattr(provider, "cpu_only"):
+                    provider_info["cpu_only"] = provider.cpu_only
+                result["providers"][name] = provider_info
             except Exception as exc:  # pragma: no cover - defensive logging
                 logger.warning("Failed to inspect embedding provider %s: %s", name, exc)
                 result["providers"][name] = {"available": False, "error": "import_failed"}
@@ -409,6 +428,8 @@ def embeddings_status() -> dict[str, Any]:
 @tool
 def build_embeddings(
     provider: str | None = None,
+    model: str | None = None,
+    cpu_only: bool | None = None,
     force: bool = False,
 ) -> dict[str, Any]:
     """Build or rebuild the embedding index for semantic search.
@@ -420,6 +441,11 @@ def build_embeddings(
         provider: Embedding provider to use: "ollama", "openai", or
                  "sentence_transformers". If None, auto-detects the best
                  available provider.
+        model: Embedding model name override. If None, uses provider default
+              or OLLAMA_MODEL env var for Ollama.
+        cpu_only: For Ollama provider, force CPU-only inference (num_gpu=0).
+                 Useful when GPU is under VRAM pressure. If None, reads from
+                 OLLAMA_CPU_ONLY env var.
         force: If True, rebuild even if embeddings already exist.
 
     Returns:
@@ -427,15 +453,23 @@ def build_embeddings(
 
     Note:
         Ollama requires a running Ollama server (configure with OLLAMA_HOST env).
+        Set OLLAMA_CPU_ONLY=true to force CPU inference.
         OpenAI requires OPENAI_API_KEY environment variable.
         SentenceTransformers requires the sentence-transformers package.
     """
     global _corpus
 
-    logger.info("build_embeddings requested provider=%s force=%s", provider, force)
+    logger.info(
+        "build_embeddings requested provider=%s model=%s cpu_only=%s force=%s",
+        provider,
+        model,
+        cpu_only,
+        force,
+    )
 
     try:
         from ifcraftcorpus.providers import (
+            EmbeddingProvider,
             OllamaEmbeddings,
             OpenAIEmbeddings,
             SentenceTransformersEmbeddings,
@@ -449,21 +483,22 @@ def build_embeddings(
         }
 
     # Get provider
-    embedding_provider = None
+    embedding_provider: EmbeddingProvider | None = None
     if provider:
-        provider_map = {
-            "ollama": OllamaEmbeddings,
-            "openai": OpenAIEmbeddings,
-            "sentence_transformers": SentenceTransformersEmbeddings,
-        }
-        if provider not in provider_map:
+        provider_lower = provider.lower()
+        if provider_lower == "ollama":
+            embedding_provider = OllamaEmbeddings(model=model, cpu_only=cpu_only)
+        elif provider_lower == "openai":
+            embedding_provider = OpenAIEmbeddings(model=model)
+        elif provider_lower == "sentence_transformers":
+            embedding_provider = SentenceTransformersEmbeddings(model=model)
+        else:
             logger.warning("Unknown embeddings provider requested: %s", provider)
             return {
                 "error": f"Unknown provider: {provider}. Use: ollama, openai, sentence_transformers"
             }
-        embedding_provider = provider_map[provider]()
     else:
-        embedding_provider = get_embedding_provider()
+        embedding_provider = get_embedding_provider(model=model, cpu_only=cpu_only)
 
     if not embedding_provider:
         logger.warning("No embedding provider available for build request")
@@ -506,7 +541,7 @@ def build_embeddings(
     # Update global corpus to use new embeddings
     _corpus = Corpus(embeddings_path=embeddings_path)
 
-    return {
+    result = {
         "status": "success",
         "items_embedded": count,
         "provider": embedding_provider.provider_name,
@@ -514,6 +549,12 @@ def build_embeddings(
         "path": str(embeddings_path),
     }
 
+    # Add cpu_only info for Ollama
+    if hasattr(embedding_provider, "cpu_only"):
+        result["cpu_only"] = embedding_provider.cpu_only
+
+    return result
+
 
 # =============================================================================
 # Subagent Prompts

{ifcraftcorpus-1.2.1 → ifcraftcorpus-1.4.0}/src/ifcraftcorpus/providers.py

@@ -9,7 +9,12 @@ Supports multiple backends:
 Provider selection via environment:
 - EMBEDDING_PROVIDER: "ollama", "openai", or "sentence-transformers"
 - OLLAMA_HOST: Ollama server URL (default: http://localhost:11434)
+- IFCRAFTCORPUS_OLLAMA_MODEL: Ollama embedding model (default: nomic-embed-text)
+- IFCRAFTCORPUS_OLLAMA_CPU_ONLY: Set to "true" or "1" to force CPU-only inference
 - OPENAI_API_KEY: OpenAI API key (required for openai provider)
+
+Note: The IFCRAFTCORPUS_ prefix avoids conflicts with other applications
+that may use Ollama with different model/GPU configurations.
 """
 
 from __future__ import annotations
@@ -113,24 +118,44 @@ class OllamaEmbeddings(EmbeddingProvider):
     Recommended model: nomic-embed-text (768 dimensions)
 
     Requires: httpx (pip install httpx)
+
+    Environment variables:
+        OLLAMA_HOST: Server URL (default: http://localhost:11434)
+        IFCRAFTCORPUS_OLLAMA_MODEL: Model name (default: nomic-embed-text)
+        IFCRAFTCORPUS_OLLAMA_CPU_ONLY: Set to "true" or "1" to force CPU-only inference
+
+    Note: The IFCRAFTCORPUS_ prefix avoids conflicts with other applications
+    that may use Ollama with different model/GPU configurations.
     """
 
     def __init__(
         self,
         model: str | None = None,
         host: str | None = None,
+        cpu_only: bool | None = None,
     ):
         """
         Initialize Ollama embeddings.
 
         Args:
-            model: Embedding model name (default: nomic-embed-text)
-            host: Ollama host URL (default: http://localhost:11434)
+            model: Embedding model name (default: nomic-embed-text, or
+                   IFCRAFTCORPUS_OLLAMA_MODEL env)
+            host: Ollama host URL (default: http://localhost:11434, or OLLAMA_HOST env)
+            cpu_only: Force CPU-only inference with num_gpu=0 (default: False, or
+                     IFCRAFTCORPUS_OLLAMA_CPU_ONLY env). Useful when GPU is under
+                     VRAM pressure or to avoid contention with other GPU workloads.
         """
-        self._model = model or DEFAULT_MODELS["ollama"]
+        self._model = model or os.getenv("IFCRAFTCORPUS_OLLAMA_MODEL") or DEFAULT_MODELS["ollama"]
         self._host = host or os.getenv("OLLAMA_HOST", "http://localhost:11434")
         self._dimension = MODEL_DIMENSIONS.get(self._model, 768)
 
+        # CPU-only mode: check parameter, then env var
+        if cpu_only is not None:
+            self._cpu_only = cpu_only
+        else:
+            env_val = os.getenv("IFCRAFTCORPUS_OLLAMA_CPU_ONLY", "").lower()
+            self._cpu_only = env_val in ("true", "1", "yes")
+
     @property
     def model(self) -> str:
         return self._model
@@ -143,20 +168,30 @@ class OllamaEmbeddings(EmbeddingProvider):
     def provider_name(self) -> str:
         return "ollama"
 
+    @property
+    def cpu_only(self) -> bool:
+        """Whether CPU-only mode is enabled."""
+        return self._cpu_only
+
     def embed(self, texts: list[str]) -> EmbeddingResult:
         """Embed texts using Ollama."""
         import httpx
 
         embeddings: list[list[float]] = []
 
+        # Build request payload
+        base_payload: dict[str, object] = {"model": self._model}
+
+        # Add options for CPU-only mode
+        if self._cpu_only:
+            base_payload["options"] = {"num_gpu": 0}
+
         with httpx.Client(timeout=60.0) as client:
             for text in texts:
+                payload = {**base_payload, "prompt": text}
                 response = client.post(
                     f"{self._host}/api/embeddings",
-                    json={
-                        "model": self._model,
-                        "prompt": text,
-                    },
+                    json=payload,
                 )
                 response.raise_for_status()
                 data = response.json()
@@ -372,6 +407,7 @@ class SentenceTransformersEmbeddings(EmbeddingProvider):
 def get_embedding_provider(
     provider_name: str | None = None,
     model: str | None = None,
+    cpu_only: bool | None = None,
 ) -> EmbeddingProvider | None:
     """
     Get an embedding provider based on configuration.
@@ -384,6 +420,8 @@ def get_embedding_provider(
     Args:
         provider_name: Explicit provider name ("ollama", "openai", "sentence-transformers")
         model: Optional model override
+        cpu_only: For Ollama, force CPU-only inference (num_gpu=0). If None, reads
+                 from OLLAMA_CPU_ONLY env var.
 
     Returns:
         Configured EmbeddingProvider or None if none available
@@ -395,7 +433,7 @@ def get_embedding_provider(
         name = name.lower()
         provider: EmbeddingProvider
         if name == "ollama":
-            provider = OllamaEmbeddings(model=model)
+            provider = OllamaEmbeddings(model=model, cpu_only=cpu_only)
         elif name == "openai":
             provider = OpenAIEmbeddings(model=model)
         elif name in ("sentence-transformers", "st", "local"):
@@ -410,9 +448,9 @@ def get_embedding_provider(
         return None
 
     # Auto-detect: try Ollama first, then OpenAI, then SentenceTransformers
-    ollama = OllamaEmbeddings(model=model)
+    ollama = OllamaEmbeddings(model=model, cpu_only=cpu_only)
     if ollama.check_availability():
-        logger.info(f"Using Ollama embeddings ({ollama.model})")
+        logger.info(f"Using Ollama embeddings ({ollama.model}, cpu_only={ollama.cpu_only})")
         return ollama
 
     openai = OpenAIEmbeddings(model=model)