iatoolkit 0.71.4__py3-none-any.whl → 0.91.1__py3-none-any.whl

iatoolkit/services/dispatcher_service.py

@@ -4,13 +4,9 @@
 # IAToolkit is open source software.
 
 from iatoolkit.common.exceptions import IAToolkitException
-from iatoolkit.services.prompt_manager_service import PromptService
-from iatoolkit.services.sql_service import SqlService
+from iatoolkit.services.prompt_service import PromptService
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
 from iatoolkit.services.configuration_service import ConfigurationService
-from iatoolkit.repositories.models import Company, Function
-from iatoolkit.services.excel_service import ExcelService
-from iatoolkit.services.mail_service import MailService
 from iatoolkit.common.util import Utility
 from injector import inject
 import logging
@@ -23,28 +19,25 @@ class Dispatcher:
                  config_service: ConfigurationService,
                  prompt_service: PromptService,
                  llmquery_repo: LLMQueryRepo,
-                 util: Utility,
-                 sql_service: SqlService,
-                 excel_service: ExcelService,
-                 mail_service: MailService):
+                 util: Utility,):
         self.config_service = config_service
         self.prompt_service = prompt_service
         self.llmquery_repo = llmquery_repo
         self.util = util
-        self.sql_service = sql_service
-        self.excel_service = excel_service
-        self.mail_service = mail_service
-        self.system_functions = _FUNCTION_LIST
-        self.system_prompts = _SYSTEM_PROMPT
 
+        self._tool_service = None
         self._company_registry = None
         self._company_instances = None
 
-        self.tool_handlers = {
-            "iat_generate_excel": self.excel_service.excel_generator,
-            "iat_send_email": self.mail_service.send_mail,
-            "iat_sql_query": self.sql_service.exec_sql
-        }
+
+    @property
+    def tool_service(self):
+        """Lazy-loads and returns the ToolService instance to avoid circular imports."""
+        if self._tool_service is None:
+            from iatoolkit import current_iatoolkit
+            from iatoolkit.services.tool_service import ToolService
+            self._tool_service = current_iatoolkit().get_injector().get(ToolService)
+        return self._tool_service
 
     @property
     def company_registry(self):
@@ -65,87 +58,49 @@ class Dispatcher:
         # initialize the system functions and prompts
         self.setup_iatoolkit_system()
 
-        """Loads the configuration of every company"""
+        # Loads the configuration of every company: company.yaml file
         for company_name, company_instance in self.company_instances.items():
             try:
                 # read company configuration from company.yaml
                 self.config_service.load_configuration(company_name, company_instance)
 
-                # register the company databases
-                self._register_company_databases(company_name)
-
             except Exception as e:
                 logging.error(f"❌ Failed to register configuration for '{company_name}': {e}")
-                continue
+                raise e
 
         return True
 
-    def _register_company_databases(self, company_name: str):
-        """
-        Reads the data_sources config for a company and registers each
-        database with the central SqlService.
-        """
-        logging.info(f"  -> Registering databases for '{company_name}'...")
-        data_sources_config = self.config_service.get_configuration(company_name, 'data_sources')
-
-        if not data_sources_config or not data_sources_config.get('sql'):
-            return
-
-        for db_config in data_sources_config['sql']:
-            db_name = db_config.get('database')
-            db_env_var = db_config.get('connection_string_env')
-
-            # resolve the URI connection string from the environment variable
-            db_uri = os.getenv(db_env_var) if db_env_var else None
-            if not db_uri:
-                logging.error(
-                    f"-> Skipping database registration for '{company_name}' due to missing 'database' name or invalid connection URI.")
-                return
-
-            self.sql_service.register_database(db_name, db_uri)
-
     def setup_iatoolkit_system(self):
-        # create system functions
-        for function in self.system_functions:
-            self.llmquery_repo.create_or_update_function(
-                Function(
-                    company_id=None,
-                    system_function=True,
-                    name=function['function_name'],
-                    description= function['description'],
-                    parameters=function['parameters']
-                )
-            )
+        try:
+            # system tools registration
+            self.tool_service.register_system_tools()
 
-        # create the system prompts
-        i = 1
-        for prompt in self.system_prompts:
-            self.prompt_service.create_prompt(
-                prompt_name=prompt['name'],
-                description=prompt['description'],
-                order=1,
-                is_system_prompt=True,
-            )
-            i += 1
+            # system prompts registration
+            self.prompt_service.register_system_prompts()
+
+        except Exception as e:
+            self.llmquery_repo.rollback()
+            raise IAToolkitException(IAToolkitException.ErrorType.DATABASE_ERROR, str(e))
 
 
-    def dispatch(self, company_short_name: str, action: str, **kwargs) -> dict:
+    def dispatch(self, company_short_name: str, function_name: str, **kwargs) -> dict:
         company_key = company_short_name.lower()
 
         if company_key not in self.company_instances:
             available_companies = list(self.company_instances.keys())
             raise IAToolkitException(
                 IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                f"Empresa '{company_short_name}' no configurada. Empresas disponibles: {available_companies}"
+                f"Company '{company_short_name}' not configured. available companies: {available_companies}"
             )
 
-        # check if action is a system function
-        if action in self.tool_handlers:
-            return  self.tool_handlers[action](**kwargs)
+        # check if action is a system function using ToolService
+        if self.tool_service.is_system_tool(function_name):
+            handler = self.tool_service.get_system_handler(function_name)
+            return handler(company_short_name, **kwargs)
 
         company_instance = self.company_instances[company_short_name]
         try:
-            return company_instance.handle_request(action, **kwargs)
+            return company_instance.handle_request(function_name, **kwargs)
         except IAToolkitException as e:
             # Si ya es una IAToolkitException, la relanzamos para preservar el tipo de error original.
             raise e
@@ -153,158 +108,9 @@ class Dispatcher:
         except Exception as e:
             logging.exception(e)
             raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                               f"Error en function call '{action}': {str(e)}") from e
+                               f"Error in function call '{function_name}': {str(e)}") from e
 
-    def get_company_services(self, company: Company) -> list[dict]:
-        # create the syntax with openai response syntax, for the company function list
-        tools = []
-        functions = self.llmquery_repo.get_company_functions(company)
-
-        for function in functions:
-            # make sure is always on
-            function.parameters["additionalProperties"] = False
-
-            ai_tool = {
-                "type": "function",
-                "name": function.name,
-                "description": function.description,
-                "parameters": function.parameters,
-                "strict": True
-            }
-            tools.append(ai_tool)
-        return tools
-
-    def get_user_info(self, company_name: str, user_identifier: str) -> dict:
-        if company_name not in self.company_instances:
-            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                                     f"company not configured: {company_name}")
-
-        # source 2: external company user
-        company_instance = self.company_instances[company_name]
-        try:
-            external_user_profile = company_instance.get_user_info(user_identifier)
-        except Exception as e:
-            logging.exception(e)
-            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                                     f"Error in get_user_info: {company_name}: {str(e)}") from e
-
-        return external_user_profile
 
     def get_company_instance(self, company_name: str):
         """Returns the instance for a given company name."""
         return self.company_instances.get(company_name)
-
-
-# iatoolkit system prompts
-_SYSTEM_PROMPT = [
-    {'name': 'query_main', 'description':'iatoolkit main prompt'},
-    {'name': 'format_styles', 'description':'output format styles'},
-    {'name': 'sql_rules', 'description':'instructions  for SQL queries'}
-]
-
-# iatoolkit  built-in functions (Tools)
-_FUNCTION_LIST = [
-    {
-        "function_name": "iat_sql_query",
-        "description": "Servicio SQL de IAToolkit: debes utilizar este servicio para todas las consultas a base de datos.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "database": {
-                    "type": "string",
-                    "description": "nombre de la base de datos a consultar: `database_name`"
-                },
-                "query": {
-                    "type": "string",
-                    "description": "string con la consulta en sql"
-                },
-            },
-            "required": ["database", "query"]
-            }
-    },
-    {
-        "function_name": "iat_generate_excel",
-        "description": "Generador de Excel."
-                    "Genera un archivo Excel (.xlsx) a partir de una lista de diccionarios. "
-                    "Cada diccionario representa una fila del archivo. "
-                    "el archivo se guarda en directorio de descargas."
-                    "retorna diccionario con filename, attachment_token (para enviar archivo por mail)"
-                    "content_type y download_link",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "filename": {
-                    "type": "string",
-                    "description": "Nombre del archivo de salida (ejemplo: 'reporte.xlsx')",
-                    "pattern": "^.+\\.xlsx?$"
-                },
-                "sheet_name": {
-                    "type": "string",
-                    "description": "Nombre de la hoja dentro del Excel",
-                    "minLength": 1
-                },
-                "data": {
-                    "type": "array",
-                    "description": "Lista de diccionarios. Cada diccionario representa una fila.",
-                    "minItems": 1,
-                    "items": {
-                        "type": "object",
-                        "properties": {},
-                        "additionalProperties": {
-                            "anyOf": [
-                                {"type": "string"},
-                                {"type": "number"},
-                                {"type": "boolean"},
-                                {"type": "null"},
-                                {
-                                    "type": "string",
-                                    "format": "date"
-                                }
-                            ]
-                        }
-                    }
-                }
-            },
-            "required": ["filename", "sheet_name", "data"]
-        }
-    },
-    {
-        'function_name': "iat_send_email",
-        'description':  "iatoolkit mail system. "        
-            "envia mails cuando un usuario lo solicita."
-            "Si no te indican quien envia el correo utiliza la dirección iatoolkit@iatoolkit.com",
-         'parameters': {
-            "type": "object",
-            "properties": {
-                "from_email": {"type": "string","description": "dirección de correo electrónico  que esta enviando el email."},
-                "recipient": {"type": "string", "description": "email del destinatario"},
-                "subject": {"type": "string", "description": "asunto del email"},
-                "body": {"type": "string", "description": "HTML del email"},
-                "attachments": {
-                    "type": "array",
-                    "description": "Lista de archivos adjuntos codificados en base64",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "filename": {
-                          "type": "string",
-                          "description": "Nombre del archivo con su extensión (ej. informe.pdf)"
-                        },
-                        "content": {
-                          "type": "string",
-                          "description": "Contenido del archivo en b64."
-                        },
-                        "attachment_token": {
-                          "type": "string",
-                          "description": "token para descargar el archivo."
-                        }
-                      },
-                      "required": ["filename", "content", "attachment_token"],
-                      "additionalProperties": False
-                    }
-                }
-            },
-            "required": ["from_email","recipient", "subject", "body", "attachments"]
-        }
-     }
-]

iatoolkit/services/document_service.py

@@ -12,10 +12,15 @@ import pytesseract
 from injector import inject
 from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.services.i18n_service import I18nService
+from iatoolkit.services.excel_service import ExcelService
+
 
 class DocumentService:
     @inject
-    def __init__(self, i18n_service: I18nService):
+    def __init__(self,
+                 excel_service: ExcelService,
+                 i18n_service: I18nService):
+        self.excel_service = excel_service
         self.i18n_service = i18n_service
 
         # max number of pages to load
@@ -40,6 +45,10 @@ class DocumentService:
                     return self.read_scanned_pdf(file_content)
                 else:
                     return self.read_pdf(file_content)
+            elif filename.lower().endswith(('.xlsx', '.xls')):
+                return self.excel_service.read_excel(file_content)
+            elif filename.lower().endswith('.csv'):
+                return self.excel_service.read_csv(file_content)
             else:
                 raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
                                    "Formato de archivo desconocido")

iatoolkit/services/embedding_service.py

@@ -5,7 +5,6 @@
 import os
 import base64
 import numpy as np
-from threading import Lock
 from huggingface_hub import InferenceClient
 from openai import OpenAI
 from injector import inject
@@ -14,12 +13,14 @@ from iatoolkit.services.i18n_service import I18nService
 from iatoolkit.repositories.profile_repo import ProfileRepo
 import logging
 
+
 # Wrapper classes to create a common interface for embedding clients
 class EmbeddingClientWrapper:
     """Abstract base class for embedding client wrappers."""
-    def __init__(self, client, model: str):
+    def __init__(self, client, model: str, dimensions: int = 1536):
         self.client = client
         self.model = model
+        self.dimensions = dimensions
 
     def get_embedding(self, text: str) -> list[float]:
         """Generates and returns an embedding for the given text."""
@@ -37,7 +38,9 @@ class OpenAIClientWrapper(EmbeddingClientWrapper):
     def get_embedding(self, text: str) -> list[float]:
         # The OpenAI API expects the input text to be clean
         text = text.replace("\n", " ")
-        response = self.client.embeddings.create(input=[text], model=self.model)
+        response = self.client.embeddings.create(input=[text],
+                                                 model=self.model,
+                                                 dimensions=self.dimensions)
         return response.data[0].embedding
 
 # Factory and Service classes
@@ -68,6 +71,7 @@ class EmbeddingClientFactory:
         if not provider:
             raise ValueError(f"Embedding provider not configured for company '{company_short_name}'.")
         model = embedding_config.get('model')
+        dimensions = int(embedding_config.get('dimensions', "1536"))
 
         api_key_name = embedding_config.get('api_key_name')
         if not api_key_name:
@@ -83,12 +87,12 @@ class EmbeddingClientFactory:
             if not model:
                 model='sentence-transformers/all-MiniLM-L6-v2'
             client = InferenceClient(model=model, token=api_key)
-            wrapper = HuggingFaceClientWrapper(client, model)
+            wrapper = HuggingFaceClientWrapper(client, model, dimensions)
         elif provider == 'openai':
             client = OpenAI(api_key=api_key)
             if not model:
                 model='text-embedding-ada-002'
-            wrapper = OpenAIClientWrapper(client, model)
+            wrapper = OpenAIClientWrapper(client, model, dimensions)
         else:
             raise NotImplementedError(f"Embedding provider '{provider}' is not implemented.")
 
@@ -126,7 +130,6 @@ class EmbeddingService:
 
             # 2. Use the wrapper's common interface to get the embedding
             embedding = client_wrapper.get_embedding(text)
-
             # 3. Process the result
             if to_base64:
                 return base64.b64encode(np.array(embedding, dtype=np.float32).tobytes()).decode('utf-8')

iatoolkit/services/excel_service.py

@@ -3,6 +3,7 @@
 #
 # IAToolkit is open source software.
 
+from flask import current_app, jsonify
 from iatoolkit.common.util import Utility
 import pandas as pd
 from uuid import uuid4
@@ -11,8 +12,9 @@ from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.services.i18n_service import I18nService
 from injector import inject
 import os
+import io
 import logging
-from flask import current_app, jsonify
+import json
 
 EXCEL_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 
@@ -25,7 +27,53 @@ class ExcelService:
         self.util = util
         self.i18n_service = i18n_service
 
-    def excel_generator(self, **kwargs) -> str:
+    def read_excel(self, file_content: bytes) -> str:
+        """
+        Reads an Excel file and converts its content to a JSON string.
+        - If the Excel file has a single sheet, it returns the JSON of that sheet.
+        - If it has multiple sheets, it returns a JSON object with sheet names as keys.
+        """
+        try:
+            # Use a BytesIO object to allow pandas to read the in-memory byte content
+            file_like_object = io.BytesIO(file_content)
+
+            # Read all sheets into a dictionary of DataFrames
+            xls = pd.read_excel(file_like_object, sheet_name=None)
+
+            if len(xls) == 1:
+                # If only one sheet, return its JSON representation directly
+                sheet_name = list(xls.keys())[0]
+                return xls[sheet_name].to_json(orient='records', indent=4)
+            else:
+                # If multiple sheets, create a dictionary of JSON strings
+                sheets_json = {}
+                for sheet_name, df in xls.items():
+                    sheets_json[sheet_name] = df.to_json(orient='records', indent=4)
+                return json.dumps(sheets_json, indent=4)
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                     self.i18n_service.t('errors.services.cannot_read_excel')) from e
+
+    def read_csv(self, file_content: bytes) -> str:
+        """
+        Reads a CSV file and converts its content to a JSON string.
+        """
+        try:
+            # Use a BytesIO object to allow pandas to read the in-memory byte content
+            file_like_object = io.BytesIO(file_content)
+
+            # Read the CSV into a DataFrame
+            df = pd.read_csv(file_like_object)
+
+            # Return JSON representation
+            return df.to_json(orient='records', indent=4)
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                     self.i18n_service.t('errors.services.cannot_read_csv')) from e
+
+    def excel_generator(self, company_short_name: str, **kwargs) -> str:
         """
         Genera un Excel a partir de una lista de diccionarios.
 

iatoolkit/services/history_manager_service.py

@@ -0,0 +1,189 @@
+import logging
+import json
+from typing import Dict, Any, Tuple, Optional
+from iatoolkit.services.user_session_context_service import UserSessionContextService
+from iatoolkit.services.i18n_service import I18nService
+from iatoolkit.infra.llm_client import llmClient
+from iatoolkit.repositories.models import Company
+from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
+from iatoolkit.repositories.profile_repo import ProfileRepo
+from injector import inject
+
+
+class HistoryManagerService:
+    """
+    Manages conversation history for LLMs in a unified way.
+    Handles:
+    1. Server-side history (e.g., OpenAI response_ids).
+    2. Client-side history (e.g., Gemini message lists).
+    3. Database persistence retrieval (full chat history).
+    """
+    TYPE_SERVER_SIDE = 'server_side'  # For models like OpenAI
+    TYPE_CLIENT_SIDE = 'client_side'  # For models like Gemini
+
+    GEMINI_MAX_TOKENS_CONTEXT_HISTORY = 200000
+
+
+    @inject
+    def __init__(self,
+                 session_context: UserSessionContextService,
+                 i18n: I18nService,
+                 llm_query_repo: LLMQueryRepo,
+                 profile_repo: ProfileRepo,
+                 llm_client: Optional[llmClient] = None):
+        self.session_context = session_context
+        self.i18n = i18n
+        self.llm_query_repo = llm_query_repo
+        self.profile_repo = profile_repo
+        self.llm_client = llm_client
+
+    def initialize_context(self,
+                           company_short_name: str,
+                           user_identifier: str,
+                           history_type: str,
+                           prepared_context: str,
+                           company: Company, model: str) -> Dict[str, Any]:
+        """
+        Initializes a new conversation history.
+        """
+        # 1. Clear existing history
+        self.session_context.clear_llm_history(company_short_name, user_identifier)
+
+        if history_type == self.TYPE_SERVER_SIDE:
+            # OpenAI: Send system prompt to API and store the resulting ID
+            response_id = self.llm_client.set_company_context(
+                company=company,
+                company_base_context=prepared_context,
+                model=model
+            )
+            self.session_context.save_last_response_id(company_short_name, user_identifier, response_id)
+            self.session_context.save_initial_response_id(company_short_name, user_identifier, response_id)
+            return {'response_id': response_id}
+
+        elif history_type == self.TYPE_CLIENT_SIDE:
+            # Gemini: Store system prompt as the first message in the list
+            context_history = [{"role": "user", "content": prepared_context}]
+            self.session_context.save_context_history(company_short_name, user_identifier, context_history)
+            return {}
+
+        return {}
+
+    def populate_request_params(self,
+                                handle: Any,
+                                user_turn_prompt: str,
+                                ignore_history: bool = False) -> bool:
+        """
+        Populates the request_params within the HistoryHandle.
+        Returns True if a rebuild is needed, False otherwise.
+        """
+        if handle.type == self.TYPE_SERVER_SIDE:
+            previous_response_id = None
+            if ignore_history:
+                previous_response_id = self.session_context.get_initial_response_id(handle.company_short_name,
+                                                                                    handle.user_identifier)
+            else:
+                previous_response_id = self.session_context.get_last_response_id(handle.company_short_name,
+                                                                                 handle.user_identifier)
+
+            if not previous_response_id:
+                handle.request_params = {}
+                return True  # Needs rebuild
+
+            handle.request_params = {'previous_response_id': previous_response_id}
+            return False
+
+        elif handle.type == self.TYPE_CLIENT_SIDE:
+            context_history = self.session_context.get_context_history(handle.company_short_name,
+                                                                       handle.user_identifier) or []
+
+            if not context_history:
+                handle.request_params = {}
+                return True  # Needs rebuild
+
+            if ignore_history and len(context_history) > 1:
+                # Keep only system prompt
+                context_history = [context_history[0]]
+
+            # For Gemini, we append the current user turn to the context sent to the API
+            context_history.append({"role": "user", "content": user_turn_prompt})
+
+            self._trim_context_history(context_history)
+
+            handle.request_params = {'context_history': context_history}
+            return False
+
+        handle.request_params = {}
+        return False
+
+    def update_history(self,
+                       history_handle: Any,
+                       user_turn_prompt: str,
+                       response: Dict[str, Any]):
+        """Saves or updates the history after a successful LLM call."""
+
+        # We access the type from the handle
+        history_type = history_handle.type
+        company_short_name = history_handle.company_short_name
+        user_identifier = history_handle.user_identifier
+
+        if history_type == self.TYPE_SERVER_SIDE:
+            if "response_id" in response:
+                self.session_context.save_last_response_id(company_short_name, user_identifier,
+                                                               response["response_id"])
+
+        elif history_type == self.TYPE_CLIENT_SIDE:
+            context_history = self.session_context.get_context_history(company_short_name,
+                                                                       user_identifier) or []
+            # Ensure the user prompt is recorded if not already.
+            # We check content equality to handle the case where the previous message was
+            # also 'user' (e.g., System Prompt) but different content.
+            last_content = context_history[-1].get("content") if context_history else None
+
+            if last_content != user_turn_prompt:
+                context_history.append({"role": "user", "content": user_turn_prompt})
+
+            if response.get('output'):
+                context_history.append({"role": "model", "content": response['output']})
+
+            self.session_context.save_context_history(company_short_name, user_identifier, context_history)
+
+    def _trim_context_history(self, context_history: list):
+        """Internal helper to keep token usage within limits for client-side history."""
+        if not context_history or len(context_history) <= 1:
+            return
+        try:
+            total_tokens = sum(self.llm_client.count_tokens(json.dumps(message)) for message in context_history)
+        except Exception as e:
+            logging.error(f"Error counting tokens for history: {e}.")
+            return
+
+        while total_tokens > self.GEMINI_MAX_TOKENS_CONTEXT_HISTORY and len(context_history) > 1:
+            try:
+                # Remove the oldest message after system prompt
+                removed_message = context_history.pop(1)
+                removed_tokens = self.llm_client.count_tokens(json.dumps(removed_message))
+                total_tokens -= removed_tokens
+                logging.warning(
+                    f"History tokens exceed limit. Removed old message. New total: {total_tokens} tokens."
+                )
+            except IndexError:
+                break
+
+    # --- Database History Management (Legacy HistoryService) ---
+
+    def get_full_history(self, company_short_name: str, user_identifier: str) -> dict:
+        """Retrieves the full persisted history from the database."""
+        try:
+            company = self.profile_repo.get_company_by_short_name(company_short_name)
+            if not company:
+                return {"error": self.i18n.t('errors.company_not_found', company_short_name=company_short_name)}
+
+            history = self.llm_query_repo.get_history(company, user_identifier)
+            if not history:
+                return {'message': 'empty history', 'history': []}
+
+            history_list = [query.to_dict() for query in history]
+            return {'message': 'history loaded ok', 'history': history_list}
+
+        except Exception as e:
+            return {'error': str(e)}