signalwire-sdk 3.6.0.dev125__tar.gz → 3.6.0.dev126__tar.gz

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## [Unreleased]
+
+### Fixed
+- Search indexes embed each chunk's text as written. They used to embed the
+  keyword-search text with the chunk's tags appended, which drops numbers and
+  repeated words: a rate-card row `| Local (10DLC) | $0.00660 | $0.00800 |`
+  was embedded as `local (10dlc)`, with every price gone. Keyword search
+  still uses the processed text, unchanged. On one corpus (23,126 chunks,
+  all-MiniLM-L6-v2, 39 labelled questions, vector ranking), MRR@10 went from
+  0.563 to 0.615. Yours may differ.
+
+### Notes for upgraders
+- Existing `.swsearch` files and pgvector collections keep working: the query
+  side is unchanged, and each index stays internally consistent. Results
+  change only after you rebuild an index.
+- To build the old way, pass `--embed-text processed` to `sw-search`, or
+  `IndexBuilder(embed_text="processed")`. New SQLite indexes record which was
+  used in their `config` table (`embed_text`), and pgvector collections in
+  their config metadata.
+
 ## [3.6.0] - 2026-10-05
 
 ### Added

{signalwire_sdk-3.6.0.dev125/signalwire/signalwire_sdk.egg-info → signalwire_sdk-3.6.0.dev126}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: signalwire-sdk
-Version: 3.6.0.dev125
+Version: 3.6.0.dev126
 Summary: SignalWire SDK for AI voice agents, live call control (RELAY) and REST. Run sw-pydocs for its documentation.
 Author-email: SignalWire Team <info@signalwire.com>
 License: MIT

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/docs/cli_guide.md

@@ -2324,6 +2324,7 @@ sw-search <source> [<source> ...] [options]
 - `--semantic-threshold FLOAT` - Threshold for semantic chunking (default: 0.5)
 - `--topic-threshold FLOAT` - Threshold for topic-based chunking (default: 0.3)
 - `--index-nlp-backend BACKEND` - NLP backend for processing: `nltk` (default) or `spacy`
+- `--embed-text {content,processed}` - What the embedding model sees: the chunk as written (`content`, default), or the keyword-search text with tags appended (`processed`), as indexes were built before this option existed
 - `--split-newlines N` - Split on N or more consecutive newlines, in addition to sentence boundaries
 - `--languages LANGS` - Comma-separated language codes (default: en)
 

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "signalwire-sdk"
-version = "3.6.0.dev125"
+version = "3.6.0.dev126"
 description = "SignalWire SDK for AI voice agents, live call control (RELAY) and REST. Run sw-pydocs for its documentation."
 authors = [
     {name = "SignalWire Team", email = "info@signalwire.com"}

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/signalwire/signalwire/__init__.py

@@ -59,7 +59,7 @@ if TYPE_CHECKING:
     from signalwire.utils.schema_utils import SchemaValidationError
     from signalwire.web import WebService
 
-__version__ = "3.6.0.dev125"
+__version__ = "3.6.0.dev126"
 
 # §6.2-python (owner decision: NO packaging split — one package, all deps required;
 # IMPORT-TIME behavior only): the agent/web symbols are lazy-imported via module

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/signalwire/signalwire/agent_server.py

@@ -68,7 +68,7 @@ class AgentServer:
         self.app = FastAPI(
             title="SignalWire AI Agents",
             description="Hosted SignalWire AI Agents",
-            version="3.6.0.dev125",
+            version="3.6.0.dev126",
             redirect_slashes=False,
             docs_url=None,
             redoc_url=None,

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/signalwire/signalwire/cli/build_search.py

@@ -319,6 +319,17 @@ Documentation: sw-pydocs prints the SDK's installed docs, examples and API.
         ),
     )
 
+    parser.add_argument(
+        "--embed-text",
+        choices=["content", "processed"],
+        default="content",
+        help=(
+            "What the embedding model sees: the chunk as written (content, "
+            "default) or the keyword-search text with tags appended (processed, "
+            "how indexes were built before this option). See IndexBuilder."
+        ),
+    )
+
     parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
 
     parser.add_argument(
@@ -519,6 +530,7 @@ def main() -> None:
                 chunk_overlap=args.overlap_size,
                 split_newlines=args.split_newlines,
                 index_nlp_backend=args.index_nlp_backend,
+                embed_text=args.embed_text,
                 verbose=args.verbose,
                 semantic_threshold=args.semantic_threshold,
                 topic_threshold=args.topic_threshold,
@@ -629,6 +641,7 @@ def main() -> None:
             chunk_overlap=args.overlap_size,
             split_newlines=args.split_newlines,
             index_nlp_backend=args.index_nlp_backend,
+            embed_text=args.embed_text,
             verbose=args.verbose,
             semantic_threshold=args.semantic_threshold,
             topic_threshold=args.topic_threshold,

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126}/signalwire/signalwire/search/index_builder.py

@@ -56,6 +56,7 @@ class IndexBuilder:
         topic_threshold: float = 0.3,
         backend: str = "sqlite",
         connection_string: str | None = None,
+        embed_text: str = "content",
     ):
         """
         Initialize the index builder
@@ -75,6 +76,26 @@ class IndexBuilder:
             topic_threshold: Similarity threshold for topic chunking (default: 0.3)
             backend: Storage backend ('sqlite' or 'pgvector') (default: 'sqlite')
             connection_string: PostgreSQL connection string for pgvector backend
+            embed_text: What the embedding model sees for each chunk.
+                'content' (default) embeds the chunk text as written.
+                'processed' embeds the keyword-search text instead -- the
+                output of preprocess_document_content with the chunk's tags
+                appended -- which is what every index built before this option
+                used.
+
+                The keyword text is built for full-text matching: it lowercases,
+                drops stopwords and numbers, and keeps each word once. That
+                suits FTS but not an embedding model, which was trained on
+                ordinary prose. A rate-card row "| Local (10DLC) | $0.00660 |
+                $0.00800 |" is embedded as "local (10dlc)", every price gone.
+
+                One measurement, on one corpus (SignalWire's support corpus,
+                23,126 chunks of docs, SDK guides, API reference, pricing and
+                articles; all-MiniLM-L6-v2; 39 hand-labelled questions; vector
+                ranking only): MRR@10 0.563 with 'processed', 0.615 with
+                'content', and 0.562 with content plus tags appended. Your
+                corpus may differ; the option exists so you can measure both.
+                Keyword search uses processed_content either way.
         """
         self.model_name = model_name
         self.chunking_strategy = chunking_strategy
@@ -89,6 +110,7 @@ class IndexBuilder:
         self.topic_threshold = topic_threshold
         self.backend = backend
         self.connection_string = connection_string
+        self.embed_text = embed_text
         # SentenceTransformer instance, loaded lazily in _load_model(); the lib
         # ships no stubs, so this is effectively Any once populated.
         self.model: Any = None
@@ -99,6 +121,11 @@ class IndexBuilder:
                 f"Invalid backend '{self.backend}'. Must be 'sqlite' or 'pgvector'"
             )
 
+        if self.embed_text not in ("content", "processed"):
+            raise ValueError(
+                f"Invalid embed_text '{self.embed_text}'. Must be 'content' or 'processed'"
+            )
+
         # Validate NLP backend
         if self.index_nlp_backend not in ["nltk", "spacy"]:
             logger.warning(
@@ -273,11 +300,14 @@ class IndexBuilder:
 
                 chunk["keywords"] = keywords
 
-                # For embedding, include tags in the text for better semantic matching
-                embedding_text = processed["enhanced_text"]
-                if chunk_tags:
-                    # Append tags to the text for embedding generation
-                    embedding_text += " " + " ".join(chunk_tags)
+                # The embedding sees the chunk as written unless asked for the
+                # keyword text; see embed_text in __init__ for why.
+                if self.embed_text == "content":
+                    embedding_text = chunk["content"]
+                else:
+                    embedding_text = processed["enhanced_text"]
+                    if chunk_tags:
+                        embedding_text += " " + " ".join(chunk_tags)
 
                 # Generate embedding (suppress progress bar)
                 embedding = self.model.encode(embedding_text, show_progress_bar=False)
@@ -707,6 +737,7 @@ class IndexBuilder:
                 "min_chunk_size": str(self.min_chunk_size),
                 "chunk_overlap": str(self.chunk_overlap),
                 "preprocessing_version": "1.0",
+                "embed_text": self.embed_text,
                 "languages": json.dumps(languages),
                 "created_at": datetime.now().isoformat(),
                 "sources": json.dumps(
@@ -930,6 +961,7 @@ class IndexBuilder:
                     "chunk_size": self.chunk_size,
                     "chunk_overlap": self.chunk_overlap,
                     "index_nlp_backend": self.index_nlp_backend,
+                    "embed_text": self.embed_text,
                 },
             }
 

{signalwire_sdk-3.6.0.dev125 → signalwire_sdk-3.6.0.dev126/signalwire/signalwire_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: signalwire-sdk
-Version: 3.6.0.dev125
+Version: 3.6.0.dev126
 Summary: SignalWire SDK for AI voice agents, live call control (RELAY) and REST. Run sw-pydocs for its documentation.
 Author-email: SignalWire Team <info@signalwire.com>
 License: MIT