iatoolkit 2.51.1__tar.gz → 2.51.3__tar.gz

{iatoolkit-2.51.1/src/iatoolkit.egg-info → iatoolkit-2.51.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.51.1
+Version: 2.51.3
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT
@@ -47,9 +47,6 @@ Requires-Dist: humanfriendly==10.0
 Requires-Dist: idna==3.10
 Requires-Dist: injector==0.22.0
 Requires-Dist: Jinja2==3.1.5
-Requires-Dist: langchain==0.3.19
-Requires-Dist: langchain-core==0.3.35
-Requires-Dist: langchain-text-splitters==0.3.6
 Requires-Dist: markdown2==2.5.3
 Requires-Dist: nh3==0.3.7
 Requires-Dist: openai==2.8.1
@@ -79,7 +76,6 @@ Requires-Dist: SQLAlchemy==2.0.49
 Requires-Dist: sqlalchemy-redshift==1.0.0
 Requires-Dist: redshift_connector==2.1.14
 Requires-Dist: tiktoken==0.8.0
-Requires-Dist: tokenizers>=0.22.0
 Requires-Dist: websocket-client==1.8.0
 Requires-Dist: websockets==14.1
 Requires-Dist: Werkzeug==3.1.3

{iatoolkit-2.51.1 → iatoolkit-2.51.3}/pyproject.toml

@@ -59,9 +59,6 @@ dependencies = [
   "idna==3.10",
   "injector==0.22.0",
   "Jinja2==3.1.5",
-  "langchain==0.3.19",
-  "langchain-core==0.3.35",
-  "langchain-text-splitters==0.3.6",
   "markdown2==2.5.3",
   "nh3==0.3.7",
   "openai==2.8.1",
@@ -91,7 +88,10 @@ dependencies = [
   "sqlalchemy-redshift==1.0.0",
   "redshift_connector==2.1.14",
   "tiktoken==0.8.0",
-  "tokenizers>=0.22.0",
+  # tokenizers is not declared: nothing here imports it, and the only
+  # package that needs it is transformers, which arrives through the
+  # docling and inference extras and pulls its own. Declaring it put 8MB
+  # of Rust bindings into every install that uses neither.
   "websocket-client==1.8.0",
   "websockets==14.1",
   "Werkzeug==3.1.3",

{iatoolkit-2.51.1 → iatoolkit-2.51.3}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.51.1"
+__version__ = "2.51.3"
 
 # Expose main classes and functions at the top level of the package
 

iatoolkit-2.51.3/src/iatoolkit/common/text_splitter.py

@@ -0,0 +1,170 @@
+"""Recursive character text splitting for the RAG ingestion path.
+
+A drop-in replacement for `langchain_text_splitters.RecursiveCharacterTextSplitter`,
+written out here because that single class was the only langchain import in the
+whole fleet and it dragged langchain-core, langsmith, zstandard and orjson into
+every deployment.
+
+The behaviour is reproduced exactly, not approximated: the chunk boundaries this
+produces decide how documents are embedded, so a corpus ingested before the swap
+and one ingested after have to agree. `src/tests/common/test_text_splitter.py`
+pins that with golden fixtures captured from the original implementation.
+
+Two details are easy to get wrong when reading the algorithm:
+
+- Separators are matched as regular expressions, but the caller passes literals,
+  so each one is `re.escape`d before use. Without that, the "." separator this
+  codebase configures would match every character.
+- `keep_separator` is True here (langchain's own default for the recursive
+  splitter, which overrides the False it inherits). The separator is attached to
+  the *start* of the following piece, and the pieces are then joined with "" -
+  not with the separator - when merging. Joining with the separator instead
+  duplicates it in every chunk.
+"""
+
+import logging
+import re
+from typing import Callable, Iterable, List, Optional
+
+
+logger = logging.getLogger(__name__)
+
+
+class RecursiveCharacterTextSplitter:
+    """Splits text by trying each separator in turn, falling back to the next.
+
+    For every stretch of text the first separator that occurs in it is used. Any
+    piece still larger than `chunk_size` is split again with the remaining
+    separators, and the pieces that do fit are merged back up to `chunk_size`
+    with `chunk_overlap` characters carried between neighbours.
+    """
+
+    def __init__(
+        self,
+        separators: Optional[List[str]] = None,
+        chunk_size: int = 4000,
+        chunk_overlap: int = 200,
+        length_function: Callable[[str], int] = len,
+        keep_separator: bool = True,
+        strip_whitespace: bool = True,
+    ) -> None:
+        if chunk_overlap > chunk_size:
+            raise ValueError(
+                f"Got a larger chunk overlap ({chunk_overlap}) than chunk size "
+                f"({chunk_size}), should be smaller."
+            )
+        self._separators = separators or ["\n\n", "\n", " ", ""]
+        self._chunk_size = chunk_size
+        self._chunk_overlap = chunk_overlap
+        self._length_function = length_function
+        self._keep_separator = keep_separator
+        self._strip_whitespace = strip_whitespace
+
+    def split_text(self, text: str) -> List[str]:
+        return self._split_text(text, self._separators)
+
+    def _split_text(self, text: str, separators: List[str]) -> List[str]:
+        final_chunks: List[str] = []
+
+        # The last separator is the fallback when none of the others occur.
+        separator = separators[-1]
+        new_separators: List[str] = []
+        for i, candidate in enumerate(separators):
+            if candidate == "":
+                separator = candidate
+                break
+            if re.search(re.escape(candidate), text):
+                separator = candidate
+                new_separators = separators[i + 1:]
+                break
+
+        splits = self._split_with_separator(text, separator)
+
+        # Pieces that already fit accumulate until one does not; that one is
+        # flushed first so the recursion cannot reorder the output.
+        good_splits: List[str] = []
+        merge_separator = "" if self._keep_separator else separator
+        for piece in splits:
+            if self._length_function(piece) < self._chunk_size:
+                good_splits.append(piece)
+                continue
+            if good_splits:
+                final_chunks.extend(self._merge_splits(good_splits, merge_separator))
+                good_splits = []
+            if not new_separators:
+                # No separator left to try: the piece goes out oversized rather
+                # than being cut mid-word.
+                final_chunks.append(piece)
+            else:
+                final_chunks.extend(self._split_text(piece, new_separators))
+        if good_splits:
+            final_chunks.extend(self._merge_splits(good_splits, merge_separator))
+        return final_chunks
+
+    def _split_with_separator(self, text: str, separator: str) -> List[str]:
+        """Splits on `separator`, keeping it attached to the piece that follows."""
+        if not separator:
+            return [c for c in list(text) if c != ""]
+
+        parts = re.split(f"({re.escape(separator)})", text)
+        if self._keep_separator:
+            # re.split with a capturing group alternates text, separator, text...
+            # Pairing each separator with the following text is what puts the
+            # separator at the start of its piece.
+            splits = [parts[i] + parts[i + 1] for i in range(1, len(parts), 2)]
+            if len(parts) % 2 == 0:
+                splits += parts[-1:]
+            splits = [parts[0]] + splits
+        else:
+            splits = re.split(re.escape(separator), text)
+        return [s for s in splits if s != ""]
+
+    def _join_docs(self, docs: List[str], separator: str) -> Optional[str]:
+        text = separator.join(docs)
+        if self._strip_whitespace:
+            text = text.strip()
+        return text or None
+
+    def _merge_splits(self, splits: Iterable[str], separator: str) -> List[str]:
+        """Packs consecutive pieces into chunks, carrying `chunk_overlap` over."""
+        separator_len = self._length_function(separator)
+
+        docs: List[str] = []
+        current_doc: List[str] = []
+        total = 0
+        for piece in splits:
+            piece_len = self._length_function(piece)
+            if (
+                total + piece_len + (separator_len if len(current_doc) > 0 else 0)
+                > self._chunk_size
+            ):
+                if total > self._chunk_size:
+                    logger.warning(
+                        "Created a chunk of size %s, which is longer than the "
+                        "specified %s",
+                        total,
+                        self._chunk_size,
+                    )
+                if len(current_doc) > 0:
+                    doc = self._join_docs(current_doc, separator)
+                    if doc is not None:
+                        docs.append(doc)
+                    # Drop from the front until what is left is small enough to
+                    # serve as the overlap for the next chunk.
+                    while total > self._chunk_overlap or (
+                        total
+                        + piece_len
+                        + (separator_len if len(current_doc) > 0 else 0)
+                        > self._chunk_size
+                        and total > 0
+                    ):
+                        total -= self._length_function(current_doc[0]) + (
+                            separator_len if len(current_doc) > 1 else 0
+                        )
+                        current_doc = current_doc[1:]
+            current_doc.append(piece)
+            total += piece_len + (separator_len if len(current_doc) > 1 else 0)
+        doc = self._join_docs(current_doc, separator)
+        if doc is not None:
+            docs.append(doc)
+        return docs

{iatoolkit-2.51.1 → iatoolkit-2.51.3}/src/iatoolkit/core.py

@@ -415,7 +415,6 @@ class IAToolkit:
     def _bind_services(self, binder: Binder):
         from iatoolkit.services.query_service import QueryService
         from iatoolkit.services.api_key_service import ApiKeyService
-        from iatoolkit.services.benchmark_service import BenchmarkService
         from iatoolkit.services.parsers.parsing_service import ParsingService
         from iatoolkit.services.parsers.provider_factory import ParsingProviderFactory
         from iatoolkit.services.parsers.provider_resolver import ParsingProviderResolver
@@ -458,7 +457,6 @@ class IAToolkit:
 
         binder.bind(QueryService, to=QueryService)
         binder.bind(ApiKeyService, to=ApiKeyService)
-        binder.bind(BenchmarkService, to=BenchmarkService)
         binder.bind(ParsingService, to=ParsingService)
         binder.bind(ParsingProviderFactory, to=ParsingProviderFactory)
         binder.bind(ParsingProviderResolver, to=ParsingProviderResolver)

{iatoolkit-2.51.1 → iatoolkit-2.51.3}/src/iatoolkit/services/knowledge_base_service.py

@@ -12,7 +12,7 @@ from iatoolkit.services.profile_service import ProfileService
 from iatoolkit.services.i18n_service import I18nService
 from iatoolkit.services.storage_service import StorageService
 from iatoolkit.services.parsers.parsing_service import ParsingService
-from langchain_text_splitters import RecursiveCharacterTextSplitter
+from iatoolkit.common.text_splitter import RecursiveCharacterTextSplitter
 from iatoolkit.services.visual_kb_service import VisualKnowledgeBaseService
 from sqlalchemy import desc, func
 from typing import Any
@@ -50,7 +50,8 @@ class KnowledgeBaseService:
         self.profile_service = profile_service
         self.i18n_service = i18n_service
 
-        # Configure LangChain for intelligent text splitting
+        # Chunking for the RAG index. These numbers decide how the corpus is
+        # embedded, so changing them means re-ingesting what is already stored.
         self.text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=1000,
             chunk_overlap=100,
@@ -83,7 +84,7 @@ class KnowledgeBaseService:
         Synchronously processes a document through the entire RAG pipeline:
         1. Saves initial metadata and raw content reference to the SQL Document table.
         2. Parses content via configured parsing provider (docling/basic/custom).
-        3. Splits the text into semantic chunks using LangChain.
+        3. Splits the text into semantic chunks.
         4. Vectorizes and saves chunks to the Vector Store (VSRepo).
         5. Updates the document status to ACTIVE or FAILED.
 

{iatoolkit-2.51.1 → iatoolkit-2.51.3/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.51.1
+Version: 2.51.3
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT
@@ -47,9 +47,6 @@ Requires-Dist: humanfriendly==10.0
 Requires-Dist: idna==3.10
 Requires-Dist: injector==0.22.0
 Requires-Dist: Jinja2==3.1.5
-Requires-Dist: langchain==0.3.19
-Requires-Dist: langchain-core==0.3.35
-Requires-Dist: langchain-text-splitters==0.3.6
 Requires-Dist: markdown2==2.5.3
 Requires-Dist: nh3==0.3.7
 Requires-Dist: openai==2.8.1
@@ -79,7 +76,6 @@ Requires-Dist: SQLAlchemy==2.0.49
 Requires-Dist: sqlalchemy-redshift==1.0.0
 Requires-Dist: redshift_connector==2.1.14
 Requires-Dist: tiktoken==0.8.0
-Requires-Dist: tokenizers>=0.22.0
 Requires-Dist: websocket-client==1.8.0
 Requires-Dist: websockets==14.1
 Requires-Dist: Werkzeug==3.1.3

{iatoolkit-2.51.1 → iatoolkit-2.51.3}/src/iatoolkit.egg-info/SOURCES.txt

@@ -20,6 +20,7 @@ src/iatoolkit/common/model_registry.py
 src/iatoolkit/common/routes.py
 src/iatoolkit/common/secret_resolver.py
 src/iatoolkit/common/session_manager.py
+src/iatoolkit/common/text_splitter.py
 src/iatoolkit/common/url_safety.py
 src/iatoolkit/common/util.py
 src/iatoolkit/common/interfaces/__init__.py
@@ -109,7 +110,6 @@ src/iatoolkit/services/__init__.py
 src/iatoolkit/services/api_key_service.py
 src/iatoolkit/services/attachment_policy_service.py
 src/iatoolkit/services/auth_service.py
-src/iatoolkit/services/benchmark_service.py
 src/iatoolkit/services/branding_service.py
 src/iatoolkit/services/company_context_service.py
 src/iatoolkit/services/configuration_service.py

{iatoolkit-2.51.1 → iatoolkit-2.51.3}/src/iatoolkit.egg-info/requires.txt

@@ -37,9 +37,6 @@ humanfriendly==10.0
 idna==3.10
 injector==0.22.0
 Jinja2==3.1.5
-langchain==0.3.19
-langchain-core==0.3.35
-langchain-text-splitters==0.3.6
 markdown2==2.5.3
 nh3==0.3.7
 openai==2.8.1
@@ -69,7 +66,6 @@ SQLAlchemy==2.0.49
 sqlalchemy-redshift==1.0.0
 redshift_connector==2.1.14
 tiktoken==0.8.0
-tokenizers>=0.22.0
 websocket-client==1.8.0
 websockets==14.1
 Werkzeug==3.1.3

iatoolkit-2.51.1/src/iatoolkit/services/benchmark_service.py

@@ -1,139 +0,0 @@
-# Copyright (c) 2024 Fernando Libedinsky
-# Product: IAToolkit
-#
-# IAToolkit is open source software.
-
-import pandas as pd
-import time
-import logging
-from injector import inject
-from iatoolkit.services.query_service import QueryService
-from iatoolkit.repositories.profile_repo import ProfileRepo
-from iatoolkit.common.exceptions import IAToolkitException
-
-
-class BenchmarkService:
-    @inject
-    def __init__(self, query_service: QueryService, profile_repo: ProfileRepo):
-        self.query_service = query_service
-        self.profile_repo = profile_repo
-
-    def _update_results_row(self, df: pd.DataFrame, index: int,
-                            status: str,
-                            context_time: float,
-                            gpt_time: float,
-                            answer: str,
-                            error_msg: str,
-                            query_id: int,
-                            stats: dict):
-        df.at[index, 'status'] = status
-        df.at[index, 'context_time'] = round(context_time, 2)
-        df.at[index, 'gpt_time'] = round(gpt_time, 2)
-        df.at[index, 'answer'] = answer
-        df.at[index, 'error_message'] = error_msg[:512]
-        df.at[index, 'query_id'] = query_id
-        df.at[index, 'in_tokens'] = stats.get('input_tokens', 0)
-        df.at[index, 'out_tokens'] = stats.get('output_tokens', 0)
-        df.at[index, 'retry'] = stats.get('sql_retry_count', 0)
-
-    def run(self, company_short_name: str, file_path: str):
-        if not file_path.endswith('.xlsx'):
-            raise IAToolkitException(IAToolkitException.ErrorType.INVALID_PARAMETER,
-                        f"solo se leer archivos .xlsx")
-
-        try:
-            df = pd.read_excel(file_path, keep_default_na=False)
-        except FileNotFoundError:
-            raise IAToolkitException(IAToolkitException.ErrorType.INVALID_NAME,
-                        f"El archivo no fue encontrado en: {file_path}")
-
-        required_columns = ['username', 'client_identity', 'prompt_name', 'question', 'model']
-        if not all(col in df.columns for col in required_columns):
-            raise IAToolkitException(IAToolkitException.ErrorType.INVALID_PARAMETER,
-                               f"La planilla debe contener las columnas: {required_columns}")
-
-        # Añadir columnas para los resultados
-        df['status'] = 'pending'
-        df['context_time'] = 0.0
-        df['gpt_time'] = 0.0
-        df['answer'] = ''
-        df['error_message'] = ''
-        df['query_id'] = 0
-        df['in_tokens'] = 0
-        df['out_tokens'] = 0
-        df['retry'] = 0
-
-        company = self.profile_repo.get_company_by_short_name(company_short_name)
-        if not company:
-            raise IAToolkitException(IAToolkitException.ErrorType.CONFIG_ERROR, f"Compañía {company_short_name} no encontrada.")
-
-        total_rows = len(df)
-        logging.info(f"Iniciando benchmark para {total_rows} casos de prueba desde el archivo: {file_path}")
-
-        for index, row in df.iterrows():
-            status = 'OK'
-            error_msg = ''
-            answer = ''
-            stats = {}
-
-            try:
-                username = str(row['username'])
-                client_identity = str(row['client_identity'])
-                prompt_name = str(row['prompt_name'])
-                question = str(row['question'])
-                model = str(row['model'])
-
-                logging.info(
-                    f"*** Procesando caso {index + 1}/{total_rows}: client_rut='{client_identity}', prompt='{prompt_name}'")
-
-                # 1. init context with user executing the test
-                start_time = time.time()
-                self.query_service.llm_init_context(company.short_name,
-                                                    external_user_id=username,
-                                                    model=model)
-                context_time = time.time() - start_time
-
-                # 2. prepare client data.
-                client_data = {'client_identity': client_identity}
-
-                # 3. execute the query
-                start_time = time.time()
-                response = self.query_service.llm_query(
-                    company_short_name=company.short_name,
-                    prompt_name=prompt_name,
-                    question=question,
-                    external_user_id=username,
-                    client_data=client_data
-                )
-                gpt_time = time.time() - start_time
-
-                # 4. process the response
-                if response.get('error') or not response.get('valid_response'):
-                    status = 'FAILED'
-                    error_msg = response.get('error_message', 'Error desconocido en la respuesta.')
-                else:
-                    answer = response.get('answer', '')
-                    stats = response.get('stats', {})
-
-                query_id = response.get('query_id', 0)
-
-            except Exception as e:
-                status = 'FAILED'
-                error_msg = f"Excepción durante la ejecución: {type(e).__name__} - {str(e)}"
-                logging.error(f"Fallo en el caso {index + 1}: {error_msg}")
-
-            finally:
-                self._update_results_row(df, index,
-                                         status=status,
-                                         context_time=context_time,
-                                         gpt_time=gpt_time,
-                                         answer=answer,
-                                         error_msg=error_msg,
-                                         stats=stats,
-                                         query_id=query_id)
-
-        output_filename = file_path.replace('.xlsx', '_results.xlsx')
-        df.to_excel(output_filename, index=False)
-        logging.info(f"Benchmark finalizado. Resultados guardados en: {output_filename}")
-
-        return output_filename