iatoolkit 0.71.4__py3-none-any.whl → 1.4.2__py3-none-any.whl

iatoolkit/services/dispatcher_service.py

@@ -4,13 +4,9 @@
 # IAToolkit is open source software.
 
 from iatoolkit.common.exceptions import IAToolkitException
-from iatoolkit.services.prompt_manager_service import PromptService
-from iatoolkit.services.sql_service import SqlService
+from iatoolkit.services.prompt_service import PromptService
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
 from iatoolkit.services.configuration_service import ConfigurationService
-from iatoolkit.repositories.models import Company, Function
-from iatoolkit.services.excel_service import ExcelService
-from iatoolkit.services.mail_service import MailService
 from iatoolkit.common.util import Utility
 from injector import inject
 import logging
@@ -23,28 +19,25 @@ class Dispatcher:
                  config_service: ConfigurationService,
                  prompt_service: PromptService,
                  llmquery_repo: LLMQueryRepo,
-                 util: Utility,
-                 sql_service: SqlService,
-                 excel_service: ExcelService,
-                 mail_service: MailService):
+                 util: Utility,):
         self.config_service = config_service
         self.prompt_service = prompt_service
         self.llmquery_repo = llmquery_repo
         self.util = util
-        self.sql_service = sql_service
-        self.excel_service = excel_service
-        self.mail_service = mail_service
-        self.system_functions = _FUNCTION_LIST
-        self.system_prompts = _SYSTEM_PROMPT
 
+        self._tool_service = None
         self._company_registry = None
         self._company_instances = None
 
-        self.tool_handlers = {
-            "iat_generate_excel": self.excel_service.excel_generator,
-            "iat_send_email": self.mail_service.send_mail,
-            "iat_sql_query": self.sql_service.exec_sql
-        }
+
+    @property
+    def tool_service(self):
+        """Lazy-loads and returns the ToolService instance to avoid circular imports."""
+        if self._tool_service is None:
+            from iatoolkit import current_iatoolkit
+            from iatoolkit.services.tool_service import ToolService
+            self._tool_service = current_iatoolkit().get_injector().get(ToolService)
+        return self._tool_service
 
     @property
     def company_registry(self):
@@ -65,87 +58,67 @@ class Dispatcher:
         # initialize the system functions and prompts
         self.setup_iatoolkit_system()
 
-        """Loads the configuration of every company"""
-        for company_name, company_instance in self.company_instances.items():
+        # Loads the configuration of every company: company.yaml file
+        for company_short_name, company_instance in self.company_instances.items():
             try:
                 # read company configuration from company.yaml
-                self.config_service.load_configuration(company_name, company_instance)
+                config, errors = self.config_service.load_configuration(company_short_name)
 
-                # register the company databases
-                self._register_company_databases(company_name)
+                '''
+                if errors:
+                    raise IAToolkitException(
+                        IAToolkitException.ErrorType.CONFIG_ERROR,
+                        'company.yaml validation errors'
+                    )
+                '''
+
+                # complement the instance self data
+                company_instance.company_short_name = company_short_name
+                company_instance.company = config.get('company')
 
             except Exception as e:
-                logging.error(f"❌ Failed to register configuration for '{company_name}': {e}")
-                continue
+                logging.error(f"❌ Failed to register configuration for '{company_short_name}': {e}")
+                raise e
 
         return True
 
-    def _register_company_databases(self, company_name: str):
-        """
-        Reads the data_sources config for a company and registers each
-        database with the central SqlService.
-        """
-        logging.info(f"  -> Registering databases for '{company_name}'...")
-        data_sources_config = self.config_service.get_configuration(company_name, 'data_sources')
-
-        if not data_sources_config or not data_sources_config.get('sql'):
-            return
-
-        for db_config in data_sources_config['sql']:
-            db_name = db_config.get('database')
-            db_env_var = db_config.get('connection_string_env')
-
-            # resolve the URI connection string from the environment variable
-            db_uri = os.getenv(db_env_var) if db_env_var else None
-            if not db_uri:
-                logging.error(
-                    f"-> Skipping database registration for '{company_name}' due to missing 'database' name or invalid connection URI.")
-                return
-
-            self.sql_service.register_database(db_name, db_uri)
-
     def setup_iatoolkit_system(self):
-        # create system functions
-        for function in self.system_functions:
-            self.llmquery_repo.create_or_update_function(
-                Function(
-                    company_id=None,
-                    system_function=True,
-                    name=function['function_name'],
-                    description= function['description'],
-                    parameters=function['parameters']
-                )
-            )
+        try:
+            # system tools registration
+            self.tool_service.register_system_tools()
 
-        # create the system prompts
-        i = 1
-        for prompt in self.system_prompts:
-            self.prompt_service.create_prompt(
-                prompt_name=prompt['name'],
-                description=prompt['description'],
-                order=1,
-                is_system_prompt=True,
-            )
-            i += 1
+            # system prompts registration
+            self.prompt_service.register_system_prompts()
+
+        except Exception as e:
+            self.llmquery_repo.rollback()
+            raise IAToolkitException(IAToolkitException.ErrorType.DATABASE_ERROR, str(e))
 
 
-    def dispatch(self, company_short_name: str, action: str, **kwargs) -> dict:
+    def dispatch(self, company_short_name: str, function_name: str, **kwargs) -> dict:
         company_key = company_short_name.lower()
 
         if company_key not in self.company_instances:
             available_companies = list(self.company_instances.keys())
             raise IAToolkitException(
                 IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                f"Empresa '{company_short_name}' no configurada. Empresas disponibles: {available_companies}"
+                f"Company '{company_short_name}' not configured. available companies: {available_companies}"
             )
 
-        # check if action is a system function
-        if action in self.tool_handlers:
-            return  self.tool_handlers[action](**kwargs)
+        # check if action is a system function using ToolService
+        if self.tool_service.is_system_tool(function_name):
+            # this is the system function to be executed.
+            handler = self.tool_service.get_system_handler(function_name)
+            logging.info(
+                f"Calling system handler [{function_name}] "
+                f"with company_short_name={company_short_name} "
+                f"and kwargs={kwargs}"
+            )
+            return handler(company_short_name, **kwargs)
 
         company_instance = self.company_instances[company_short_name]
         try:
-            return company_instance.handle_request(action, **kwargs)
+            return company_instance.handle_request(function_name, **kwargs)
         except IAToolkitException as e:
             # Si ya es una IAToolkitException, la relanzamos para preservar el tipo de error original.
             raise e
@@ -153,158 +126,9 @@ class Dispatcher:
         except Exception as e:
             logging.exception(e)
             raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                               f"Error en function call '{action}': {str(e)}") from e
+                               f"Error in function call '{function_name}': {str(e)}") from e
 
-    def get_company_services(self, company: Company) -> list[dict]:
-        # create the syntax with openai response syntax, for the company function list
-        tools = []
-        functions = self.llmquery_repo.get_company_functions(company)
-
-        for function in functions:
-            # make sure is always on
-            function.parameters["additionalProperties"] = False
-
-            ai_tool = {
-                "type": "function",
-                "name": function.name,
-                "description": function.description,
-                "parameters": function.parameters,
-                "strict": True
-            }
-            tools.append(ai_tool)
-        return tools
-
-    def get_user_info(self, company_name: str, user_identifier: str) -> dict:
-        if company_name not in self.company_instances:
-            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                                     f"company not configured: {company_name}")
-
-        # source 2: external company user
-        company_instance = self.company_instances[company_name]
-        try:
-            external_user_profile = company_instance.get_user_info(user_identifier)
-        except Exception as e:
-            logging.exception(e)
-            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
-                                     f"Error in get_user_info: {company_name}: {str(e)}") from e
-
-        return external_user_profile
 
     def get_company_instance(self, company_name: str):
         """Returns the instance for a given company name."""
         return self.company_instances.get(company_name)
-
-
-# iatoolkit system prompts
-_SYSTEM_PROMPT = [
-    {'name': 'query_main', 'description':'iatoolkit main prompt'},
-    {'name': 'format_styles', 'description':'output format styles'},
-    {'name': 'sql_rules', 'description':'instructions  for SQL queries'}
-]
-
-# iatoolkit  built-in functions (Tools)
-_FUNCTION_LIST = [
-    {
-        "function_name": "iat_sql_query",
-        "description": "Servicio SQL de IAToolkit: debes utilizar este servicio para todas las consultas a base de datos.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "database": {
-                    "type": "string",
-                    "description": "nombre de la base de datos a consultar: `database_name`"
-                },
-                "query": {
-                    "type": "string",
-                    "description": "string con la consulta en sql"
-                },
-            },
-            "required": ["database", "query"]
-            }
-    },
-    {
-        "function_name": "iat_generate_excel",
-        "description": "Generador de Excel."
-                    "Genera un archivo Excel (.xlsx) a partir de una lista de diccionarios. "
-                    "Cada diccionario representa una fila del archivo. "
-                    "el archivo se guarda en directorio de descargas."
-                    "retorna diccionario con filename, attachment_token (para enviar archivo por mail)"
-                    "content_type y download_link",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "filename": {
-                    "type": "string",
-                    "description": "Nombre del archivo de salida (ejemplo: 'reporte.xlsx')",
-                    "pattern": "^.+\\.xlsx?$"
-                },
-                "sheet_name": {
-                    "type": "string",
-                    "description": "Nombre de la hoja dentro del Excel",
-                    "minLength": 1
-                },
-                "data": {
-                    "type": "array",
-                    "description": "Lista de diccionarios. Cada diccionario representa una fila.",
-                    "minItems": 1,
-                    "items": {
-                        "type": "object",
-                        "properties": {},
-                        "additionalProperties": {
-                            "anyOf": [
-                                {"type": "string"},
-                                {"type": "number"},
-                                {"type": "boolean"},
-                                {"type": "null"},
-                                {
-                                    "type": "string",
-                                    "format": "date"
-                                }
-                            ]
-                        }
-                    }
-                }
-            },
-            "required": ["filename", "sheet_name", "data"]
-        }
-    },
-    {
-        'function_name': "iat_send_email",
-        'description':  "iatoolkit mail system. "        
-            "envia mails cuando un usuario lo solicita."
-            "Si no te indican quien envia el correo utiliza la dirección iatoolkit@iatoolkit.com",
-         'parameters': {
-            "type": "object",
-            "properties": {
-                "from_email": {"type": "string","description": "dirección de correo electrónico  que esta enviando el email."},
-                "recipient": {"type": "string", "description": "email del destinatario"},
-                "subject": {"type": "string", "description": "asunto del email"},
-                "body": {"type": "string", "description": "HTML del email"},
-                "attachments": {
-                    "type": "array",
-                    "description": "Lista de archivos adjuntos codificados en base64",
-                    "items": {
-                      "type": "object",
-                      "properties": {
-                        "filename": {
-                          "type": "string",
-                          "description": "Nombre del archivo con su extensión (ej. informe.pdf)"
-                        },
-                        "content": {
-                          "type": "string",
-                          "description": "Contenido del archivo en b64."
-                        },
-                        "attachment_token": {
-                          "type": "string",
-                          "description": "token para descargar el archivo."
-                        }
-                      },
-                      "required": ["filename", "content", "attachment_token"],
-                      "additionalProperties": False
-                    }
-                }
-            },
-            "required": ["from_email","recipient", "subject", "body", "attachments"]
-        }
-     }
-]

iatoolkit/services/document_service.py

@@ -12,10 +12,15 @@ import pytesseract
 from injector import inject
 from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.services.i18n_service import I18nService
+from iatoolkit.services.excel_service import ExcelService
+
 
 class DocumentService:
     @inject
-    def __init__(self, i18n_service: I18nService):
+    def __init__(self,
+                 excel_service: ExcelService,
+                 i18n_service: I18nService):
+        self.excel_service = excel_service
         self.i18n_service = i18n_service
 
         # max number of pages to load
@@ -40,6 +45,10 @@ class DocumentService:
                     return self.read_scanned_pdf(file_content)
                 else:
                     return self.read_pdf(file_content)
+            elif filename.lower().endswith(('.xlsx', '.xls')):
+                return self.excel_service.read_excel(file_content)
+            elif filename.lower().endswith('.csv'):
+                return self.excel_service.read_csv(file_content)
             else:
                 raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
                                    "Formato de archivo desconocido")

iatoolkit/services/embedding_service.py

@@ -5,7 +5,6 @@
 import os
 import base64
 import numpy as np
-from threading import Lock
 from huggingface_hub import InferenceClient
 from openai import OpenAI
 from injector import inject
@@ -14,12 +13,14 @@ from iatoolkit.services.i18n_service import I18nService
 from iatoolkit.repositories.profile_repo import ProfileRepo
 import logging
 
+
 # Wrapper classes to create a common interface for embedding clients
 class EmbeddingClientWrapper:
     """Abstract base class for embedding client wrappers."""
-    def __init__(self, client, model: str):
+    def __init__(self, client, model: str, dimensions: int = 1536):
         self.client = client
         self.model = model
+        self.dimensions = dimensions
 
     def get_embedding(self, text: str) -> list[float]:
         """Generates and returns an embedding for the given text."""
@@ -37,7 +38,9 @@ class OpenAIClientWrapper(EmbeddingClientWrapper):
     def get_embedding(self, text: str) -> list[float]:
         # The OpenAI API expects the input text to be clean
         text = text.replace("\n", " ")
-        response = self.client.embeddings.create(input=[text], model=self.model)
+        response = self.client.embeddings.create(input=[text],
+                                                 model=self.model,
+                                                 dimensions=self.dimensions)
         return response.data[0].embedding
 
 # Factory and Service classes
@@ -68,6 +71,7 @@ class EmbeddingClientFactory:
         if not provider:
             raise ValueError(f"Embedding provider not configured for company '{company_short_name}'.")
         model = embedding_config.get('model')
+        dimensions = int(embedding_config.get('dimensions', "1536"))
 
         api_key_name = embedding_config.get('api_key_name')
         if not api_key_name:
@@ -83,12 +87,12 @@ class EmbeddingClientFactory:
             if not model:
                 model='sentence-transformers/all-MiniLM-L6-v2'
             client = InferenceClient(model=model, token=api_key)
-            wrapper = HuggingFaceClientWrapper(client, model)
+            wrapper = HuggingFaceClientWrapper(client, model, dimensions)
         elif provider == 'openai':
             client = OpenAI(api_key=api_key)
             if not model:
                 model='text-embedding-ada-002'
-            wrapper = OpenAIClientWrapper(client, model)
+            wrapper = OpenAIClientWrapper(client, model, dimensions)
         else:
             raise NotImplementedError(f"Embedding provider '{provider}' is not implemented.")
 
@@ -126,7 +130,6 @@ class EmbeddingService:
 
             # 2. Use the wrapper's common interface to get the embedding
             embedding = client_wrapper.get_embedding(text)
-
             # 3. Process the result
             if to_base64:
                 return base64.b64encode(np.array(embedding, dtype=np.float32).tobytes()).decode('utf-8')

iatoolkit/services/excel_service.py

@@ -3,6 +3,7 @@
 #
 # IAToolkit is open source software.
 
+from flask import current_app, jsonify
 from iatoolkit.common.util import Utility
 import pandas as pd
 from uuid import uuid4
@@ -11,8 +12,9 @@ from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.services.i18n_service import I18nService
 from injector import inject
 import os
+import io
 import logging
-from flask import current_app, jsonify
+import json
 
 EXCEL_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 
@@ -25,7 +27,53 @@ class ExcelService:
         self.util = util
         self.i18n_service = i18n_service
 
-    def excel_generator(self, **kwargs) -> str:
+    def read_excel(self, file_content: bytes) -> str:
+        """
+        Reads an Excel file and converts its content to a JSON string.
+        - If the Excel file has a single sheet, it returns the JSON of that sheet.
+        - If it has multiple sheets, it returns a JSON object with sheet names as keys.
+        """
+        try:
+            # Use a BytesIO object to allow pandas to read the in-memory byte content
+            file_like_object = io.BytesIO(file_content)
+
+            # Read all sheets into a dictionary of DataFrames
+            xls = pd.read_excel(file_like_object, sheet_name=None)
+
+            if len(xls) == 1:
+                # If only one sheet, return its JSON representation directly
+                sheet_name = list(xls.keys())[0]
+                return xls[sheet_name].to_json(orient='records', indent=4)
+            else:
+                # If multiple sheets, create a dictionary of JSON strings
+                sheets_json = {}
+                for sheet_name, df in xls.items():
+                    sheets_json[sheet_name] = df.to_json(orient='records', indent=4)
+                return json.dumps(sheets_json, indent=4)
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                     self.i18n_service.t('errors.services.cannot_read_excel')) from e
+
+    def read_csv(self, file_content: bytes) -> str:
+        """
+        Reads a CSV file and converts its content to a JSON string.
+        """
+        try:
+            # Use a BytesIO object to allow pandas to read the in-memory byte content
+            file_like_object = io.BytesIO(file_content)
+
+            # Read the CSV into a DataFrame
+            df = pd.read_csv(file_like_object)
+
+            # Return JSON representation
+            return df.to_json(orient='records', indent=4)
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                     self.i18n_service.t('errors.services.cannot_read_csv')) from e
+
+    def excel_generator(self, company_short_name: str, **kwargs) -> str:
         """
         Genera un Excel a partir de una lista de diccionarios.
 

iatoolkit/services/file_processor_service.py

@@ -74,9 +74,6 @@ class FileProcessor:
                 if not self._apply_filters(file_name):
                     continue
 
-                if self.config.echo:
-                    print(f'loading: {file_name}')
-
                 content = self.connector.get_file_content(file_path)
 
                 # execute the callback function
@@ -87,8 +84,6 @@ class FileProcessor:
                                      context=self.config.context)
                 self.processed_files += 1
 
-                logging.info(f"Successfully processed file: {file_path}")
-
             except Exception as e:
                 logging.error(f"Error processing {file_path}: {e}")
                 if not self.config.continue_on_error: