ifcraftcorpus 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

ifcraftcorpus/index.py

@@ -56,21 +56,58 @@ from ifcraftcorpus.parser import Document, parse_directory
 def _sanitize_fts_query(query: str) -> str:
     """Sanitize a query string for the FTS5 MATCH clause.
 
-    This function replaces hyphens with spaces to prevent FTS5 from
-    interpreting them as the `NOT` operator. This is intended to correctly
-    handle natural language queries with hyphenated words, for example
-    transforming "haunted-house" into a search for "haunted house".
+    This function replaces special characters that could cause FTS5 syntax
+    errors with spaces. This is intended to correctly handle natural language
+    queries from LLMs and users, for example transforming:
+    - "haunted-house" into "haunted house" (hyphen as NOT operator)
+    - "dialogue, subtext" into "dialogue subtext" (comma syntax error)
 
     It also collapses any resulting multiple spaces into a single space.
 
+    Note: This is used as a fallback when raw FTS5 queries fail. The search
+    method first tries the raw query to support advanced FTS5 syntax, then
+    falls back to sanitized query on syntax errors.
+    See https://github.com/pvliesdonk/if-craft-corpus/issues/10
+
     Args:
         query: Raw query string from user input.
 
     Returns:
         Sanitized query safe for FTS5 MATCH.
     """
-    # Replace hyphens and collapse whitespace in one go.
-    return " ".join(query.replace("-", " ").split())
+    # Replace problematic characters with spaces:
+    # - hyphen: FTS5 interprets as NOT operator
+    # - comma: FTS5 column list syntax
+    # - parentheses: FTS5 grouping syntax
+    # - curly braces: FTS5 column filter syntax
+    # - caret: FTS5 position marker
+    # - plus: FTS5 column weight
+    # - colon after words could affect column queries, but we preserve it
+    #   to allow intentional column:value syntax
+    # Using str.translate is more efficient for replacing multiple single characters.
+    translation_table = str.maketrans("-,(){}^+", " " * 8)
+    sanitized = query.translate(translation_table)
+    # Collapse whitespace
+    return " ".join(sanitized.split())
+
+
+def _is_fts5_query_error(error: sqlite3.OperationalError) -> bool:
+    """Check if an OperationalError is an FTS5 query parsing error.
+
+    Args:
+        error: The SQLite OperationalError to check.
+
+    Returns:
+        True if this is an FTS5 query error that might be recoverable
+        by sanitizing the query.
+    """
+    msg = str(error).lower()
+    # FTS5 syntax errors (e.g., "fts5: syntax error near ','")
+    if "fts5" in msg and "syntax error" in msg:
+        return True
+    # Column errors from FTS5 query parsing (e.g., "no such column: voice")
+    # This can happen when hyphens are interpreted as column filters
+    return "no such column" in msg
 
 
 @dataclass
@@ -359,51 +396,25 @@ class CorpusIndex:
             self.add_document(doc)
         return len(documents)
 
-    def search(
+    def _execute_fts_query(
         self,
-        query: str,
-        *,
-        cluster: str | None = None,
-        limit: int = 10,
+        fts_query: str,
+        cluster: str | None,
+        limit: int,
     ) -> list[SearchResult]:
-        """Search the corpus using FTS5 full-text search.
-
-        Performs a keyword search using SQLite FTS5 with BM25 ranking.
-        Supports the full FTS5 query syntax for advanced searches.
+        """Execute an FTS5 query and return results.
 
         Args:
-            query: Search query. Supports FTS5 syntax:
-
-                - Simple keywords: ``dialogue``
-                - Phrases: ``"character voice"``
-                - Boolean: ``tension AND suspense``, ``horror NOT comedy``
-                - Prefix: ``narrat*``
-                - Column-specific: ``title:craft``
-
-            cluster: Optional cluster name to filter results. Only returns
-                matches from the specified cluster.
-            limit: Maximum number of results to return. Default 10.
+            fts_query: The FTS5 query string.
+            cluster: Optional cluster filter.
+            limit: Maximum results to return.
 
         Returns:
-            List of :class:`SearchResult` objects, sorted by BM25 relevance
-            score in descending order (best matches first).
+            List of SearchResult objects.
 
-        Example:
-            >>> # Simple search
-            >>> results = index.search("dialogue")
-
-            >>> # Phrase search
-            >>> results = index.search('"character voice"')
-
-            >>> # Boolean with cluster filter
-            >>> results = index.search("tension OR suspense",
-            ...                        cluster="emotional-design",
-            ...                        limit=5)
+        Raises:
+            sqlite3.OperationalError: If the query has invalid FTS5 syntax.
         """
-        # Build FTS5 query - sanitize to handle special characters
-        fts_query = _sanitize_fts_query(query)
-
-        # Add cluster filter if specified
         where_clause = ""
         params: list[str | int] = [fts_query]
         if cluster:
@@ -447,6 +458,72 @@ class CorpusIndex:
 
         return results
 
+    def search(
+        self,
+        query: str,
+        *,
+        cluster: str | None = None,
+        limit: int = 10,
+    ) -> list[SearchResult]:
+        """Search the corpus using FTS5 full-text search.
+
+        Performs a keyword search using SQLite FTS5 with BM25 ranking.
+        Supports the full FTS5 query syntax for advanced searches.
+
+        The search first attempts to execute the query as-is to support
+        advanced FTS5 syntax. If that fails with a syntax error, it falls
+        back to a sanitized version of the query that treats special
+        characters as word separators.
+
+        Args:
+            query: Search query. Supports FTS5 syntax:
+
+                - Simple keywords: ``dialogue``
+                - Phrases: ``"character voice"``
+                - Boolean: ``tension AND suspense``, ``horror NOT comedy``
+                - Prefix: ``narrat*``
+                - Column-specific: ``title:craft``
+
+                Natural language queries with punctuation (e.g., "dialogue,
+                subtext") are also supported - they will be automatically
+                sanitized if they cause syntax errors.
+
+            cluster: Optional cluster name to filter results. Only returns
+                matches from the specified cluster.
+            limit: Maximum number of results to return. Default 10.
+
+        Returns:
+            List of :class:`SearchResult` objects, sorted by BM25 relevance
+            score in descending order (best matches first).
+
+        Example:
+            >>> # Simple search
+            >>> results = index.search("dialogue")
+
+            >>> # Phrase search
+            >>> results = index.search('"character voice"')
+
+            >>> # Boolean with cluster filter
+            >>> results = index.search("tension OR suspense",
+            ...                        cluster="emotional-design",
+            ...                        limit=5)
+
+            >>> # Natural language (auto-sanitized)
+            >>> results = index.search("dialogue, subtext")
+        """
+        # Try raw query first to support advanced FTS5 syntax
+        try:
+            return self._execute_fts_query(query, cluster, limit)
+        except sqlite3.OperationalError as e:
+            if not _is_fts5_query_error(e):
+                raise
+
+        # Fallback to sanitized query for natural language input
+        sanitized = _sanitize_fts_query(query)
+        if not sanitized:
+            return []
+        return self._execute_fts_query(sanitized, cluster, limit)
+
     def list_documents(self) -> list[dict[str, str]]:
         """List all indexed documents with their metadata.
 

ifcraftcorpus/mcp_server.py

@@ -194,9 +194,24 @@ def search_corpus(
     worldbuilding, and genre conventions.
 
     Args:
-        query: Search query describing what craft guidance you need.
-               Examples: "dialogue subtext", "branching narrative",
-               "pacing action scenes", "horror atmosphere".
+        query: Search query. Supports natural language or FTS5 syntax:
+
+               Natural language examples:
+               - "dialogue subtext"
+               - "branching narrative"
+               - "pacing action scenes"
+
+               FTS5 advanced syntax:
+               - Exact phrases: '"character voice"'
+               - Boolean NOT: "dialogue NOT comedy"
+               - Boolean OR: "tension OR suspense"
+               - Boolean AND: "dialogue AND subtext"
+               - Prefix search: "narrat*"
+               - Column filter: "title:craft", "cluster:genre-conventions"
+
+               Natural language queries with punctuation are automatically
+               sanitized, so both styles work seamlessly.
+
         cluster: Optional topic cluster to filter by. Valid clusters:
                  narrative-structure, prose-and-language, genre-conventions,
                  audience-and-access, world-and-setting, emotional-design,