iatoolkit 0.3.1__py3-none-any.whl

services/dispatcher_service.py

@@ -0,0 +1,312 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Producto: IAToolkit
+# Todos los derechos reservados.
+# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
+
+from iatoolkit import current_iatoolkit
+from common.exceptions import IAToolkitException
+from services.prompt_manager_service import PromptService
+from repositories.llm_query_repo import LLMQueryRepo
+from repositories.models import Company, Function
+from services.excel_service import ExcelService
+from services.mail_service import MailService
+from iatoolkit.company_registry import get_company_registry
+from common.util import Utility
+from injector import inject
+import logging
+import os
+
+
+class Dispatcher:
+    @inject
+    def __init__(self,
+                 prompt_service: PromptService,
+                 llmquery_repo: LLMQueryRepo,
+                 util: Utility,
+                 excel_service: ExcelService,
+                 mail_service: MailService):
+        self.prompt_service = prompt_service
+        self.llmquery_repo = llmquery_repo
+        self.util = util
+        self.excel_service = excel_service
+        self.mail_service = mail_service
+        self.system_functions = _FUNCTION_LIST
+        self.system_prompts = _SYSTEM_PROMPT
+
+        # Use the global registry
+        self.company_registry = get_company_registry()
+
+        # The dispatcher starts "empty" and will be initialized later.
+        self.company_classes = {}
+        self.initialize_companies()
+
+        self.tool_handlers = {
+            "iat_generate_excel": self.excel_service.excel_generator,
+            "iat_send_email": self.mail_service.send_mail,
+        }
+
+    def initialize_companies(self):
+        """
+        Initializes and instantiates all registered company classes.
+        This method should be called *after* the main injector is fully configured.
+        """
+        if self.company_classes: # Prevent re-initialization
+            return
+
+        # ✅ NOW it is safe to get the injector and instantiate companies.
+        injector = current_iatoolkit()._get_injector()
+        self.company_registry.set_injector(injector)
+        self.company_classes = self.company_registry.instantiate_companies()
+
+    def start_execution(self):
+        """Runs the startup logic for all registered companies."""
+        # Ensure companies are initialized before starting them
+        if not self.company_classes:
+            self.initialize_companies()
+
+        for company_name, company_instance in self.company_classes.items():
+            logging.info(f'Starting execution for company: {company_name}')
+            company_instance.start_execution()
+
+        return True
+
+    def init_db(self):
+        # create system functions
+        for function in self.system_functions:
+            self.llmquery_repo.create_or_update_function(
+                Function(
+                    company_id=None,
+                    system_function=True,
+                    name=function['function_name'],
+                    description= function['description'],
+                    parameters=function['parameters']
+                )
+            )
+
+            # create the system prompts
+            i = 1
+            for prompt in self.system_prompts:
+                self.prompt_service.create_prompt(
+                    prompt_name=prompt['name'],
+                    description=prompt['description'],
+                    order=1,
+                    is_system_prompt=True
+                )
+                i += 1
+
+        # initialize the database for every company class
+        for company in self.company_classes.values():
+            print(f'inicializando clase: {company.__class__.__name__}')
+            company.init_db()
+
+
+    def dispatch(self, company_name: str, action: str, **kwargs) -> str:
+        company_key = company_name.lower()
+
+        if company_key not in self.company_classes:
+            available_companies = list(self.company_classes.keys())
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                f"Empresa '{company_name}' no configurada. Empresas disponibles: {available_companies}"
+            )
+
+        # check if action is a system function
+        if action in self.tool_handlers:
+            return  self.tool_handlers[action](**kwargs)
+
+        company_instance = self.company_classes[company_name]
+        try:
+            return company_instance.handle_request(action, **kwargs)
+        except IAToolkitException as e:
+            # Si ya es una IAToolkitException, la relanzamos para preservar el tipo de error original.
+            raise e
+
+        except Exception as e:
+            logging.exception(e)
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Error en function call '{action}': {str(e)}") from e
+
+    def get_company_context(self, company_name: str, **kwargs) -> str:
+        if company_name not in self.company_classes:
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Empresa no configurada: {company_name}")
+
+        company_context = ''
+
+        # read the company context from this list of markdown files,
+        # company brief, credits, operation description, etc.
+        context_dir = os.path.join(os.getcwd(), f'companies/{company_name}/context')
+        context_files = self.util.get_files_by_extension(context_dir, '.md', return_extension=True)
+        for file in context_files:
+            filepath = os.path.join(context_dir, file)
+            company_context += self.util.load_markdown_context(filepath)
+
+        # add the schemas for every table or function call responses
+        schema_dir = os.path.join(os.getcwd(), f'companies/{company_name}/schema')
+        schema_files = self.util.get_files_by_extension(schema_dir, '.yaml', return_extension=True)
+        for file in schema_files:
+            schema_name = file.split('_')[0]
+            filepath = os.path.join(schema_dir, file)
+            company_context += self.util.generate_context_for_schema(schema_name, filepath)
+
+        company_instance = self.company_classes[company_name]
+        try:
+            return company_context + company_instance.get_company_context(**kwargs)
+        except Exception as e:
+            logging.exception(e)
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Error en get_company_context de {company_name}: {str(e)}") from e
+
+    def get_company_services(self, company: Company) -> list[dict]:
+        # create the syntax with openai response syntax, for the company function list
+        tools = []
+        functions = self.llmquery_repo.get_company_functions(company)
+
+        for function in functions:
+            # make sure is always on
+            function.parameters["additionalProperties"] = False
+
+            ai_tool = {
+                "type": "function",
+                "name": function.name,
+                "description": function.description,
+                "parameters": function.parameters,
+                "strict": True
+            }
+            tools.append(ai_tool)
+        return tools
+
+    def get_user_info(self, company_name: str, **kwargs) -> dict:
+        if company_name not in self.company_classes:
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Empresa no configurada: {company_name}")
+
+        company_instance = self.company_classes[company_name]
+        try:
+            return company_instance.get_user_info(**kwargs)
+        except Exception as e:
+            logging.exception(e)
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Error en get_user_info de {company_name}: {str(e)}") from e
+
+
+    def get_metadata_from_filename(self, company_name: str, filename: str) -> dict:
+        if company_name not in self.company_classes:
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Empresa no configurada: {company_name}")
+
+        company_instance = self.company_classes[company_name]
+        try:
+            return company_instance.get_metadata_from_filename(filename)
+        except Exception as e:
+            logging.exception(e)
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Error en get_metadata_from_filename de {company_name}: {str(e)}") from e
+
+    def get_registered_companies(self) -> dict:
+        """Obtiene todas las empresas registradas (para debugging/admin)"""
+        return {
+            "registered_classes": list(self.company_registry.get_registered_companies().keys()),
+            "instantiated": list(self.company_classes.keys()),
+            "count": len(self.company_classes)
+        }
+
+
+# iatoolkit system prompts
+_SYSTEM_PROMPT = [
+    {'name': 'query_main', 'description':'main prompt de iatoolkit'},
+    {'name': 'format_styles', 'description':'formatos y estilos de salida'},
+    {'name': 'sql_rules', 'description':'instrucciones para generar sql'}
+]
+
+
+# iatoolkit  function calls
+_FUNCTION_LIST = [
+    {
+        "name": "iat_generate_excel",
+        "description": "Generador de Excel."
+                    "Genera un archivo Excel (.xlsx) a partir de una lista de diccionarios. "
+                    "Cada diccionario representa una fila del archivo. "
+                    "el archivo se guarda en directorio de descargas."
+                    "retorna diccionario con filename, attachment_token (para enviar archivo por mail)"
+                    "content_type y download_link",
+        "function_name": "iat_generate_excel",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "filename": {
+                    "type": "string",
+                    "description": "Nombre del archivo de salida (ejemplo: 'reporte.xlsx')",
+                    "pattern": "^.+\\.xlsx?$"
+                },
+                "sheet_name": {
+                    "type": "string",
+                    "description": "Nombre de la hoja dentro del Excel",
+                    "minLength": 1
+                },
+                "data": {
+                    "type": "array",
+                    "description": "Lista de diccionarios. Cada diccionario representa una fila.",
+                    "minItems": 1,
+                    "items": {
+                        "type": "object",
+                        "properties": {},
+                        "additionalProperties": {
+                            "anyOf": [
+                                {"type": "string"},
+                                {"type": "number"},
+                                {"type": "boolean"},
+                                {"type": "null"},
+                                {
+                                    "type": "string",
+                                    "format": "date"
+                                }
+                            ]
+                        }
+                    }
+                }
+            },
+            "required": ["filename", "sheet_name", "data"]
+        }
+    },
+    {
+        'name': 'Envio de mails',
+         'description':  "iatoolkit mail system. "        
+            "envia mails cuando un usuario lo solicita."
+            "Si no te indican quien envia el correo utiliza la dirección iatoolkit@iatoolkit.com",
+         'function_name': "iat_send_email",
+         'parameters': {
+            "type": "object",
+            "properties": {
+                "from_email": {"type": "string","description": "dirección de correo electrónico  que esta enviando el email."},
+                "recipient": {"type": "string", "description": "email del destinatario"},
+                "subject": {"type": "string", "description": "asunto del email"},
+                "body": {"type": "string", "description": "HTML del email"},
+                "attachments": {
+                    "type": "array",
+                    "description": "Lista de archivos adjuntos codificados en base64",
+                    "items": {
+                      "type": "object",
+                      "properties": {
+                        "filename": {
+                          "type": "string",
+                          "description": "Nombre del archivo con su extensión (ej. informe.pdf)"
+                        },
+                        "content": {
+                          "type": "string",
+                          "description": "Contenido del archivo en b64."
+                        },
+                        "attachment_token": {
+                          "type": "string",
+                          "description": "token para descargar el archivo."
+                        }
+                      },
+                      "required": ["filename", "content", "attachment_token"],
+                      "additionalProperties": False
+                    }
+                }
+            },
+            "required": ["from_email","recipient", "subject", "body", "attachments"]
+        }
+     }
+]

services/document_service.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Producto: IAToolkit
+# Todos los derechos reservados.
+# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
+
+from docx import Document
+import fitz  # PyMuPDF
+from PIL import Image
+import io
+import os
+import pytesseract
+from injector import inject
+from common.exceptions import IAToolkitException
+
+class DocumentService:
+    @inject
+    def __init__(self):
+        # max number of pages to load
+        self.max_doc_pages = int(os.getenv("MAX_DOC_PAGES", "10"))
+
+    def file_to_txt(self, filename, file_content):
+        try:
+            if filename.lower().endswith('.docx'):
+                return self.read_docx(file_content)
+            elif filename.lower().endswith('.txt'):
+                if isinstance(file_content, bytes):
+                    try:
+                        # decode using UTF-8
+                        file_content = file_content.decode('utf-8')
+                    except UnicodeDecodeError:
+                        raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                           "El archivo no es texto o la codificación no es UTF-8")
+
+                return file_content
+            elif filename.lower().endswith('.pdf'):
+                if self.is_scanned_pdf(file_content):
+                    return self.read_scanned_pdf(file_content)
+                else:
+                    return self.read_pdf(file_content)
+            else:
+                raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                   "Formato de archivo desconocido")
+        except IAToolkitException as e:
+            # Si es una excepción conocida, simplemente la relanzamos
+            raise
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
+                                   f"Error processing file: {e}") from e
+
+    def read_docx(self, file_content):
+        try:
+            # Crear un archivo en memoria desde el contenido en bytes
+            file_like_object = io.BytesIO(file_content)
+            doc = Document(file_like_object)
+
+            # to Markdown
+            md_content = ""
+            for para in doc.paragraphs:
+                # headings ...
+                if para.style.name.startswith("Heading"):
+                    level = int(para.style.name.replace("Heading ", ""))
+                    md_content += f"{'#' * level} {para.text}\n\n"
+                # lists ...
+                elif para.style.name in ["List Bullet", "List Paragraph"]:
+                    md_content += f"- {para.text}\n"
+                elif para.style.name in ["List Number"]:
+                    md_content += f"1. {para.text}\n"
+                # normal text
+                else:
+                    md_content += f"{para.text}\n\n"
+            return md_content
+        except Exception as e:
+            raise ValueError(f"Error reading .docx file: {e}")
+
+    def read_pdf(self, file_content):
+        try:
+            with fitz.open(stream=file_content, filetype="pdf") as pdf:
+                text = ""
+                for page in pdf:
+                    text += page.get_text()
+                return text
+        except Exception as e:
+            raise ValueError(f"Error reading .pdf file: {e}")
+
+    # Determina  es un documento escaneado (imagen) o contiene prompt_llm.txt seleccionable.
+    def is_scanned_pdf(self, file_content):
+        doc = fitz.open(stream=io.BytesIO(file_content), filetype='pdf')
+
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+
+            # Intenta extraer prompt_llm.txt directamente
+            text = page.get_text()
+            if text.strip():  # Si hay prompt_llm.txt, no es escaneado
+                return False
+
+            # Busca imágenes en la página
+            images = page.get_images(full=True)
+            if images:  # Si hay imágenes pero no hay prompt_llm.txt, puede ser un escaneo
+                continue
+
+        # Si no se encontró prompt_llm.txt en ninguna página
+        return True
+
+    def read_scanned_pdf(self, file_content):
+        images = self.pdf_to_images(file_content)
+        if not images:
+            return ''
+
+        document_text = ''
+        for image in images:
+            document_text += self.image_to_text(image)
+
+        return document_text
+
+    def pdf_to_images(self, file_content):
+        images = []             # list of images to return
+
+        pdf_document = fitz.open(stream=io.BytesIO(file_content), filetype='pdf')
+        if pdf_document.page_count > self.max_doc_pages:
+            return None
+
+        for page_number in range(len(pdf_document)):
+            page = pdf_document[page_number]
+
+            images_on_page = page.get_images(full=True)  # Obtiene todas las imágenes de la página
+            for img in images_on_page:
+                xref = img[0]  # Referencia de la imagen en el PDF
+                pix = fitz.Pixmap(pdf_document, xref)  # Crear el Pixmap de la imagen
+
+                # Si la imagen está en CMYK, conviértela a RGB para mayor compatibilidad
+                if pix.n > 4:  # CMYK tiene más de 4 canales
+                    pix = fitz.Pixmap(fitz.csRGB, pix)
+
+                images.append(pix)
+
+        pdf_document.close()
+        return images
+
+    def image_to_text(self, image):
+        # Determinar el modo PIL en base a pix.n
+        if image.n == 1:
+            pil_mode = "L"
+        elif image.n == 2:
+            pil_mode = "LA"
+        elif image.n == 3:
+            pil_mode = "RGB"
+        elif image.n == 4:
+            pil_mode = "RGBA"
+        else:
+            # Caso especial (conversion previa debería evitarlos)
+            raise ValueError(f"Canales desconocidos: {image.n}")
+
+        img = Image.frombytes(pil_mode, (image.width, image.height), image.samples)
+        return pytesseract.image_to_string(img, lang="spa")
+
+
+
+

services/excel_service.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Producto: IAToolkit
+# Todos los derechos reservados.
+# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
+
+from common.util import Utility
+import pandas as pd
+from uuid import uuid4
+from pathlib import Path
+from common.exceptions import IAToolkitException
+from injector import inject
+import os
+import logging
+from flask import current_app, jsonify
+
+EXCEL_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+
+class ExcelService:
+    @inject
+    def __init__(self,util: Utility):
+        self.util = util
+
+    def excel_generator(self, **kwargs) -> str:
+        """
+                Genera un Excel a partir de una lista de diccionarios.
+
+                Parámetros esperados en kwargs:
+                  - filename: str (nombre lógico a mostrar, ej. "reporte_clientes.xlsx") [obligatorio]
+                  - data: list[dict] (filas del excel) [obligatorio]
+                  - sheet_name: str = "hoja 1"
+
+                Retorna:
+                     {
+                        "filename": "reporte.xlsx",
+                        "attachment_token": "8b7f8a66-...-c1c3.xlsx",
+                        "content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                        "download_link": "/download/8b7f8a66-...-c1c3.xlsx"
+                        }
+                """
+        try:
+            # get the parameters
+            fname = kwargs.get('filename')
+            if not fname:
+                return 'falta el nombre del archivo de salida'
+
+            data = kwargs.get('data')
+            if not data or not isinstance(data, list):
+                return 'faltan los datos  o no es una lista de diccionarios'
+
+            sheet_name = kwargs.get('sheet_name', 'hoja 1')
+
+            # 1. convert dictionary to dataframe
+            df = pd.DataFrame(data)
+
+            # 3. create temporary name
+            token = f"{uuid4()}.xlsx"
+            filepath = Path("static/temp") / token
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+
+            # 4. save excel file in temporary directory
+            df.to_excel(filepath, index=False, sheet_name=sheet_name)
+
+            # 5. return the  link to the LLM
+            return {
+                "filename": fname,
+                "attachment_token": token,
+                "content_type": EXCEL_MIME,
+                "download_link": f"/download/{token}"
+                }
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.CALL_ERROR,
+                               'error generating excel file') from e
+
+    def validate_file_access(self, filename):
+        try:
+            if not filename:
+                return jsonify({"error": "Nombre de archivo inválido"})
+            # Prevent path traversal attacks
+            if '..' in filename or filename.startswith('/') or '\\' in filename:
+                return jsonify({"error": "Nombre de archivo inválido"})
+
+            temp_dir = os.path.join(current_app.root_path, 'static', 'temp')
+            file_path = os.path.join(temp_dir, filename)
+
+            if not os.path.exists(file_path):
+                return jsonify({"error": "Archivo no encontrado"})
+
+            if not os.path.isfile(file_path):
+                return jsonify({"error": "La ruta no corresponde a un archivo"})
+
+            return None
+
+        except Exception as e:
+            error_msg = f"Error validando acceso al archivo {filename}: {str(e)}"
+            logging.error(error_msg)
+            return jsonify({"error": "Error validando archivo"})

services/file_processor_service.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Producto: IAToolkit
+# Todos los derechos reservados.
+# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
+
+from infra.connectors.file_connector import FileConnector
+import logging
+import os
+from typing import Optional, Callable, Dict
+
+
+class FileProcessorConfig:
+    def __init__(
+        self,
+        filters: Dict,
+        action: Callable[[str, bytes], None],
+        continue_on_error: bool = True,
+        log_file: str = 'file_processor.log',
+        echo: bool = False,
+        context: dict = None,
+    ):
+        self.filters = filters
+        self.action = action
+        self.continue_on_error = continue_on_error
+        self.log_file = log_file
+        self.echo = echo
+        self.context = context or {}
+
+class FileProcessor:
+    def __init__(self,
+                 connector: FileConnector,
+                 config: FileProcessorConfig,
+                 logger: Optional[logging.Logger] = None):
+        self.connector = connector
+        self.config = config
+        self.logger = logger or self._setup_logger()
+        self.processed_files = 0
+
+    def _setup_logger(self):
+        logging.basicConfig(
+            filename=self.config.log_file,
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
+        return logging.getLogger(__name__)
+
+    def process_files(self):
+        try:
+            files = self.connector.list_files()
+        except Exception as e:
+            self.logger.error(f"Error fetching files: {e}")
+            return False
+
+        if self.config.echo:
+            print(f'cargando un total de {len(files)} archivos')
+
+        for file_info in files:
+            file_path = file_info['path']
+            file_name = file_info['name']
+
+            try:
+                if not self._apply_filters(file_name):
+                    continue
+
+                if self.config.echo:
+                    print(f'loading: {file_name}')
+
+                content = self.connector.get_file_content(file_path)
+
+                # execute the action defined
+                filename = os.path.basename(file_name)
+                self.config.action(filename, content, self.config.context)
+                self.processed_files += 1
+
+                self.logger.info(f"Successfully processed file: {file_path}")
+
+            except Exception as e:
+                self.logger.error(f"Error processing {file_path}: {e}")
+                if not self.config.continue_on_error:
+                    raise e
+
+    def _apply_filters(self, file_path: str) -> bool:
+        filters = self.config.filters
+
+        if 'filename_contains' in filters and filters['filename_contains'] not in file_path:
+            return False
+
+        if 'custom_filter' in filters and callable(filters['custom_filter']):
+            if not filters['custom_filter'](file_path):
+                return False
+
+        return True