iatoolkit 2.61.1__tar.gz → 2.62.0__tar.gz

{iatoolkit-2.61.1/src/iatoolkit.egg-info → iatoolkit-2.62.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.61.1
+Version: 2.62.0
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.61.1"
+__version__ = "2.62.0"
 
 # Expose main classes and functions at the top level of the package
 

iatoolkit-2.62.0/src/iatoolkit/common/search_terms.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+"""The words the hybrid search looks for (BL-005, Fase 4).
+
+The word branch searches the chunks' full text with Postgres' 'simple'
+configuration, which has no stopwords: a question would match every chunk with
+"de" or "the". So the question is reduced here to the words that mean something,
+and Postgres ORs them (see VSRepo). Exact terms the model passes as `keywords`
+take the place of the question.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Spanish and English function words. Only what never identifies a document:
+# no domain words, no numbers.
+STOPWORDS = frozenset("""
+a al algo algun alguna algunas alguno algunos ante antes aquel aquella aquello aqui asi aun cada como con
+contra cual cuales cualquier cuando cuanto de del desde donde dos el ella ellas ellos en entre era eran es
+esa esas ese eso esos esta estaba estado estan estar estas este esto estos fue fueron ha hay hace hacer hacia
+han hasta la las le les lo los mas me mi mis mucho muy nada ni no nos nosotros o otra otras otro otros para
+pero poco por porque que quien quienes se sea ser si sido sin sobre son su sus tambien tan tanto te tiene
+tienen todo todos tu tus un una unas uno unos usted ustedes ya yo
+cómo cuál cuáles cuándo cuánto dónde qué quién quiénes también aquí así aún más está están sí tú
+a an and are as at be been but by can could did do does for from had has have how i if in into is it its
+me my no not of on or our should so than that the their them then there these they this those to was we
+were what when where which who why will with would you your
+""".split())
+
+# Leading and trailing punctuation that is never part of a term. Inner
+# punctuation is kept: AX-2241, 76.123.456-7, v3.2.
+_EDGE_PUNCTUATION = "¿?¡!.,;:\"'()[]{}«»“”‘’…"
+_MAX_TERMS = 24
+_MAX_TERM_CHARS = 64
+
+
+def search_terms(query_text: str | None, keywords: str | list | None = None) -> str:
+    """
+    The text the word branch looks for: the keywords when given, else the query
+    without stopwords. Empty when nothing is left, and the caller then searches
+    by meaning only.
+    """
+    if keywords:
+        raw = keywords if isinstance(keywords, list) else [keywords]
+        source = " ".join(str(item) for item in raw if str(item or "").strip())
+        return _clean(source, drop_stopwords=False)
+    return _clean(str(query_text or ""), drop_stopwords=True)
+
+
+def _clean(text: str, *, drop_stopwords: bool) -> str:
+    terms: list[str] = []
+    seen: set[str] = set()
+    for token in re.split(r"\s+", text):
+        term = token.strip(_EDGE_PUNCTUATION)
+        if not term or not any(char.isalnum() for char in term):
+            continue
+        if drop_stopwords and term.lower() in STOPWORDS:
+            continue
+        term = term[:_MAX_TERM_CHARS]
+        if term.lower() in seen:
+            continue
+        seen.add(term.lower())
+        terms.append(term)
+        if len(terms) >= _MAX_TERMS:
+            break
+    return " ".join(terms)

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/config/system_tools_pack.yaml

@@ -280,6 +280,14 @@ tools:
                   - value
                 additionalProperties: false
           description: "Opcional en la práctica. Usa null cuando no necesites filtrar por metadata. No inventes filtros ni envíes [] por defecto. Ejemplo: [{'key':'doc.type','value':'invoice'},{'key':'chunk.source_type','value':'table'}]."
+        keywords:
+          anyOf:
+            - type: "null"
+            - type: string
+            - type: array
+              items:
+                type: string
+          description: "Opcional. Términos exactos que deben aparecer en el texto: códigos, números de contrato, RUT, nombres propios o siglas (ej. [\"AX-2241\", \"Pérez\"]). La búsqueda combina el significado de `query` con estos términos. Usa null si la pregunta no tiene términos exactos."
       required:
         - query
   - function_name: iat_document_list

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/repositories/models.py

@@ -8,7 +8,7 @@ import enum
 from sqlalchemy import Column, Integer, BigInteger, String, DateTime, Enum, Text, JSON, Boolean, ForeignKey, Table, MetaData
 from sqlalchemy.orm import DeclarativeBase
 from sqlalchemy.orm import relationship, class_mapper
-from sqlalchemy.sql import func
+from sqlalchemy.sql import func, literal_column
 from sqlalchemy import UniqueConstraint, Index
 from sqlalchemy.dialects.postgresql import JSONB
 from datetime import datetime
@@ -690,6 +690,20 @@ class VSDoc(Base):
     embedding = Column(Vector, nullable=False)
     company = relationship("Company", back_populates="vsdocs")
 
+    # Full-text index for the hybrid search's word branch (BL-005, Fase 4): an
+    # expression index, no extra column, so the table is not rewritten. The
+    # 'simple' configuration does not stem, so codes such as AX-2241 or names
+    # such as Pérez are indexed as written. Postgres only; existing databases get
+    # it from iatoolkit-enterprise/migrations/2026-10-15-vsdocs-text-index.sql,
+    # since create_all() never adds an index to a table that already exists.
+    __table_args__ = (
+        Index(
+            "ix_iat_vsdocs_text_simple_tsv",
+            func.to_tsvector(literal_column("'simple'::regconfig"), text),
+            postgresql_using="gin",
+        ).ddl_if(dialect="postgresql"),
+    )
+
     def to_dict(self):
         return {column.key: getattr(self, column.key) for column in class_mapper(self.__class__).columns}
 

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/repositories/vs_repo.py

@@ -14,6 +14,7 @@ from sqlalchemy import text
 from injector import inject
 
 from iatoolkit.common.exceptions import IAToolkitException
+from iatoolkit.common.search_terms import search_terms
 from iatoolkit.repositories.database_manager import DatabaseManager
 from iatoolkit.repositories.models import Company, VSDoc, VSImage
 from iatoolkit.services.embedding_service import EmbeddingService
@@ -96,6 +97,14 @@ class VSRepo:
             query = query.limit(max(int(limit), 0))
         return list(query.all())
 
+    SEARCH_MODE_VECTOR = "vector"
+    SEARCH_MODE_HYBRID = "hybrid"
+    SEARCH_MODES = (SEARCH_MODE_VECTOR, SEARCH_MODE_HYBRID)
+    # Each branch of the hybrid search brings this many candidates to the fusion.
+    HYBRID_CANDIDATES = 40
+    # Reciprocal Rank Fusion constant: a chunk adds 1 / (RRF_K + rank) per list.
+    RRF_K = 60
+
     def query(self,
               company_short_name: str,
               query_text: str,
@@ -103,6 +112,8 @@ class VSRepo:
               metadata_filter=None,
               collection_ids: List[int] | None = None,
               include_urls: bool = True,
+              search_mode: str = SEARCH_MODE_VECTOR,
+              keywords: str | list | None = None,
               ) -> list[Dict]:
         """
         search documents similar to the query for a company
@@ -113,6 +124,10 @@ class VSRepo:
             n_results: max number of results to return
             metadata_filter:  (e.g., {"document_type": "certificate"})
             include_urls: Generate storage URLs for returned chunks when true.
+            search_mode: "vector" (by meaning) or "hybrid" (by meaning and by
+                words, fused by rank; BL-005, Fase 4).
+            keywords: exact terms for the hybrid search's word branch, instead
+                of the query's own words.
 
         Returns:
             list of documents matching the query and filters
@@ -214,11 +229,18 @@ class VSRepo:
             sql_query_parts.extend(metadata_sql)
             params.update(metadata_params)
 
-            # join all the query parts
-            sql_query = "".join(sql_query_parts)
+            text_terms = search_terms(query_text, keywords) if search_mode == self.SEARCH_MODE_HYBRID else ""
+            hybrid = bool(text_terms)
+            if hybrid:
+                params["text_terms"] = text_terms
+                params["candidates"] = max(int(n_results or 5), self.HYBRID_CANDIDATES)
+                sql_query = self._hybrid_sql(sql_query_parts, distance_expression)
+            else:
+                # join all the query parts
+                sql_query = "".join(sql_query_parts)
 
-            # add sorting and limit of results
-            sql_query += " ORDER BY distance ASC LIMIT :n_results"
+                # add sorting and limit of results
+                sql_query += " ORDER BY distance ASC LIMIT :n_results"
 
             logging.debug(f"Executing SQL query: {sql_query}")
             logging.debug(f"With parameters: {params}")
@@ -231,6 +253,7 @@ class VSRepo:
             rows = result.fetchall()
             vector_sql_duration_ms = round((perf_counter() - vector_sql_started_at) * 1000)
             vs_documents = []
+            matches = {"both": 0, "vector": 0, "text": 0}
 
             for row in rows:
                 # create the document object with the data
@@ -238,6 +261,12 @@ class VSRepo:
                 chunk_meta = row[6] if len(row) > 6 and row[6] is not None else {}
 
                 distance = float(row[7]) if len(row) > 7 and row[7] is not None else None
+                hybrid_fields = {}
+                if hybrid:
+                    in_vector, in_text = bool(row[9]), bool(row[10])
+                    match = "both" if in_vector and in_text else "vector" if in_vector else "text"
+                    matches[match] += 1
+                    hybrid_fields = {"match": match, "rrf_score": float(row[8]) if row[8] is not None else None}
 
                 # get the url of the document
                 storage_key = row[3] if len(row) > 3 and row[3] is not None else None
@@ -257,17 +286,21 @@ class VSRepo:
                         'distance': distance,
                         'distance_metric': distance_metric,
                         'score': self._score_from_l2_distance(distance),
+                        'search_mode': self.SEARCH_MODE_HYBRID if hybrid else self.SEARCH_MODE_VECTOR,
+                        **hybrid_fields,
                     }
                 )
 
             logging.info(
-                "RAG vector query completed: company=%s collection_ids=%s dimensions=%s n_results=%s rows=%s "
-                "embedding_ms=%s company_lookup_ms=%s vector_sql_ms=%s total_ms=%s",
+                "RAG %s query completed: company=%s collection_ids=%s dimensions=%s n_results=%s rows=%s "
+                "matches=%s embedding_ms=%s company_lookup_ms=%s vector_sql_ms=%s total_ms=%s",
+                self.SEARCH_MODE_HYBRID if hybrid else self.SEARCH_MODE_VECTOR,
                 company_short_name,
                 normalized_collection_ids,
                 embedding_dimensions,
                 n_results,
                 len(rows),
+                matches if hybrid else None,
                 embedding_duration_ms,
                 company_lookup_duration_ms,
                 vector_sql_duration_ms,
@@ -288,6 +321,79 @@ class VSRepo:
         finally:
             self.session.close()
 
+    def _hybrid_sql(self, sql_query_parts: list[str], distance_expression: str) -> str:
+        """
+        Hybrid search in one statement: the nearest chunks by meaning and the
+        best chunks by words (Postgres full text, 'simple' configuration, the
+        terms OR'ed), each ranked, fused by Reciprocal Rank Fusion.
+
+        `sql_query_parts` is the vector query with every filter (company,
+        dimension, collections, metadata); both branches reuse its WHERE, so they
+        see exactly the same chunks. The word branch has its own expression, the
+        one the GIN index ix_iat_vsdocs_text_simple_tsv is built on.
+        """
+        base = "".join(sql_query_parts)
+        _, where = base.split("WHERE", 1)
+        # The dimension condition is for the vector index; the word branch does
+        # not need it (a chunk is a chunk whatever its embedding size).
+        text_where = re.sub(r"\s+AND vector_dims\(iat_vsdocs\.embedding\) = \d+", "", where)
+        tsvector = "to_tsvector('simple'::regconfig, iat_vsdocs.text)"
+        # plainto_tsquery parses the terms exactly as the index parsed the text
+        # (and cannot be broken by any character); its ANDs become ORs.
+        tsquery = "replace(plainto_tsquery('simple'::regconfig, :text_terms)::text, ' & ', ' | ')::tsquery"
+        return f"""
+            WITH vector_hits AS (
+                SELECT id, distance, ROW_NUMBER() OVER (ORDER BY distance ASC) AS rank
+                FROM (
+                    SELECT iat_vsdocs.id, ({distance_expression}) AS distance
+                    FROM iat_vsdocs, iat_documents
+                    WHERE {where}
+                    ORDER BY distance ASC
+                    LIMIT :candidates
+                ) nearest
+            ),
+            text_hits AS (
+                SELECT id, ROW_NUMBER() OVER (ORDER BY text_rank DESC, id ASC) AS rank
+                FROM (
+                    SELECT iat_vsdocs.id, ts_rank_cd({tsvector}, terms.q) AS text_rank
+                    FROM iat_vsdocs, iat_documents, (SELECT {tsquery} AS q) terms
+                    WHERE {text_where}
+                      AND {tsvector} @@ terms.q
+                    ORDER BY text_rank DESC
+                    LIMIT :candidates
+                ) best
+            ),
+            fused AS (
+                SELECT id,
+                       SUM(1.0 / ({self.RRF_K} + rank)) AS rrf_score,
+                       MIN(distance) AS distance,
+                       BOOL_OR(source = 'vector') AS in_vector,
+                       BOOL_OR(source = 'text') AS in_text
+                FROM (
+                    SELECT id, rank, distance, 'vector' AS source FROM vector_hits
+                    UNION ALL
+                    SELECT id, rank, NULL::float AS distance, 'text' AS source FROM text_hits
+                ) both_lists
+                GROUP BY id
+            )
+            SELECT iat_vsdocs.id,
+                   iat_documents.filename,
+                   iat_vsdocs.text,
+                   iat_documents.storage_key,
+                   iat_documents.meta,
+                   iat_documents.id,
+                   iat_vsdocs.meta,
+                   fused.distance,
+                   fused.rrf_score,
+                   fused.in_vector,
+                   fused.in_text
+            FROM fused
+            JOIN iat_vsdocs ON iat_vsdocs.id = fused.id
+            JOIN iat_documents ON iat_documents.id = iat_vsdocs.document_id
+            ORDER BY fused.rrf_score DESC, fused.distance ASC NULLS LAST
+            LIMIT :n_results
+        """
+
     def query_images(self,
                      company_short_name: str,
                      query_text: str,

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/services/configuration_service.py

@@ -1059,6 +1059,10 @@ class ConfigurationService:
         if kb_config and not isinstance(kb_config, dict):
             add_error("knowledge_base", "Section must be a dictionary.")
         elif kb_config:
+            search_mode = kb_config.get("search_mode")
+            if search_mode is not None and str(search_mode).strip().lower() not in {"vector", "hybrid"}:
+                add_error("knowledge_base.search_mode", "Must be 'vector' or 'hybrid'.")
+
             parsing_provider = kb_config.get("parsing_provider")
             allowed_parsing_providers = {"auto", "docling", "basic"}
             if parsing_provider is not None:

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/services/knowledge_base_service.py

@@ -491,9 +491,15 @@ class KnowledgeBaseService:
                collection: str | list[str] | None = None,
                metadata_filter: dict = None,
                include_urls: bool = True,
+               keywords: str | list | None = None,
+               search_mode: str | None = None,
                ):
         """
-        Performs a semantic search and returns the list of Document objects (chunks).
+        Searches the company's chunks and returns them (as dicts).
+
+        By meaning and by words, fused, unless the company sets
+        `knowledge_base.search_mode: vector` (BL-005, Fase 4). `keywords` are
+        exact terms for the word part; `search_mode` overrides the company's.
         Useful for UI displays where structured data is needed instead of a raw string context.
 
         Args:
@@ -536,10 +542,33 @@ class KnowledgeBaseService:
             metadata_filter=metadata_filter,
             collection_ids=collection_ids,
             include_urls=include_urls,
+            search_mode=self.resolve_search_mode(company_short_name, search_mode),
+            keywords=keywords,
         )
 
         return chunk_list
 
+    DEFAULT_SEARCH_MODE = "hybrid"
+    SEARCH_MODES = ("vector", "hybrid")
+
+    def resolve_search_mode(self, company_short_name: str, requested: str | None = None) -> str:
+        """The requested mode, else company.yaml's knowledge_base.search_mode, else hybrid."""
+        candidate = str(requested or "").strip().lower()
+        if candidate in self.SEARCH_MODES:
+            return candidate
+        try:
+            from iatoolkit import current_iatoolkit
+            from iatoolkit.services.configuration_service import ConfigurationService
+
+            # Resolved here, not injected: ConfigurationService builds this service
+            # to sync the collections when a company's config loads.
+            kb_config = current_iatoolkit().get_injector().get(ConfigurationService).get_configuration(
+                company_short_name, "knowledge_base") or {}
+        except Exception:
+            kb_config = {}
+        configured = str((kb_config or {}).get("search_mode") or "").strip().lower() if isinstance(kb_config, dict) else ""
+        return configured if configured in self.SEARCH_MODES else self.DEFAULT_SEARCH_MODE
+
     @staticmethod
     def _normalize_search_collection(collection: str | list[str] | None) -> str | list[str] | None:
         if collection is None:

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/services/tool_service.py

@@ -185,13 +185,16 @@ class ToolService:
                                      collection: str | list[str] | None = None,
                                      metadata_filter=None,
                                      n_results: int = 5,
+                                     keywords=None,
                                      **kwargs):
+        search_kwargs = {"keywords": keywords} if keywords else {}
         chunks = self.knowledge_base_service.search(
             company_short_name=company_short_name,
             query=query,
             n_results=n_results,
             collection=collection,
             metadata_filter=metadata_filter,
+            **search_kwargs,
         )
         typed_chunks = self._normalize_chunks_for_tool(chunks)
 

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit/views/rag_api_view.py

@@ -310,23 +310,33 @@ class RagApiView(MethodView):
             n_results = int(data.get('k', 5))
             collection = data.get('collection')
             metadata_filter = data.get('metadata_filter')
+            # The Search Lab can compare modes; empty means the company's own.
+            search_mode = str(data.get('search_mode') or '').strip().lower() or None
+            keywords = data.get('keywords') or None
 
             if not query:
                 msg = self.i18n_service.t('rag.search.query_required')
                 return jsonify({'result': 'error', 'message': msg}), 400
 
             # 3. Call Service
+            extra = {}
+            if search_mode:
+                extra['search_mode'] = search_mode
+            if keywords:
+                extra['keywords'] = keywords
             chunks = self.knowledge_base_service.search(
                 company_short_name=company_short_name,
                 query=query,
                 n_results=n_results,
                 collection=collection,
                 metadata_filter=metadata_filter,
+                **extra,
             )
 
             return jsonify({
                 "result": "success",
-                "chunks": chunks
+                "chunks": chunks,
+                "search_mode": chunks[0].get("search_mode") if chunks else None,
             }), 200
 
         except IAToolkitException as e:

{iatoolkit-2.61.1 → iatoolkit-2.62.0/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.61.1
+Version: 2.62.0
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.61.1 → iatoolkit-2.62.0}/src/iatoolkit.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ src/iatoolkit/common/company_paths.py
 src/iatoolkit/common/exceptions.py
 src/iatoolkit/common/model_registry.py
 src/iatoolkit/common/routes.py
+src/iatoolkit/common/search_terms.py
 src/iatoolkit/common/secret_resolver.py
 src/iatoolkit/common/session_manager.py
 src/iatoolkit/common/text_splitter.py