iatoolkit 0.3.9__py3-none-any.whl → 0.107.4__py3-none-any.whl

services/load_documents_service.py

@@ -1,212 +0,0 @@
-# Copyright (c) 2024 Fernando Libedinsky
-# Producto: IAToolkit
-# Todos los derechos reservados.
-# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
-
-from repositories.vs_repo import VSRepo
-from repositories.document_repo import DocumentRepo
-from repositories.profile_repo import ProfileRepo
-from repositories.llm_query_repo import LLMQueryRepo
-from repositories.models import Document, VSDoc, Company
-from services.document_service import DocumentService
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from infra.connectors.file_connector_factory import FileConnectorFactory
-from services.file_processor_service import FileProcessorConfig, FileProcessor
-from services.dispatcher_service import Dispatcher
-from common.exceptions import IAToolkitException
-import logging
-import base64
-from injector import inject
-from typing import Dict
-
-
-class LoadDocumentsService:
-    @inject
-    def __init__(self,
-                 file_connector_factory: FileConnectorFactory,
-                 doc_service: DocumentService,
-                 doc_repo: DocumentRepo,
-                 vector_store: VSRepo,
-                 profile_repo: ProfileRepo,
-                 dispatcher: Dispatcher,
-                 llm_query_repo: LLMQueryRepo
-                 ):
-        self.doc_service = doc_service
-        self.doc_repo = doc_repo
-        self.profile_repo = profile_repo
-        self.llm_query_repo = llm_query_repo
-        self.vector_store = vector_store
-        self.file_connector_factory = file_connector_factory
-        self.dispatcher = dispatcher
-        self.company = None
-
-        # lower warnings
-        logging.getLogger().setLevel(logging.ERROR)
-
-        self.splitter = RecursiveCharacterTextSplitter(
-            chunk_size=1000,
-            chunk_overlap=100,
-            separators=["\n\n", "\n", "."]
-        )
-
-    # load the files for all of the companies.
-    def load(self, doc_type: str = None):
-        # doc_type: an optional document_type for loading
-        files_loaded = 0
-        companies = self.profile_repo.get_companies()
-
-        for company in companies:
-            load_config = company.parameters.get('load', {})
-            if not load_config:
-                continue
-
-            print(f"Cargando datos de ** {company.short_name} **")
-            self.company = company
-
-            # Si hay configuraciones de tipos de documento específicos
-            doc_types_config = load_config.get('document_types', {})
-
-            if doc_types_config and len(doc_types_config) > 0:
-                # Si se especificó un tipo de documento, cargar solo ese tipo
-                if doc_type and doc_type in doc_types_config:
-                    files_loaded += self._load_document_type(company, doc_type, doc_types_config[doc_type])
-                # Si no se especificó, cargar todos los tipos configurados
-                elif not doc_type:
-                    for type_name, type_config in doc_types_config.items():
-                        files_loaded += self._load_document_type(company, type_name, type_config)
-            else:
-                # Comportamiento anterior: usar la configuración general
-                connector = load_config.get('connector', {})
-                if not connector:
-                    raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
-                                       f"Falta configurar conector en empresa {company.short_name}")
-
-                files_loaded += self.load_data_source(connector)
-
-        return {'message': f'{files_loaded} files processed'}
-
-    def _load_document_type(self, company: Company, doc_type_name: str, type_config: Dict) -> int:
-        # load specific document_types for a company
-        connector = type_config.get('connector')
-        if not connector:
-            logging.warning(f"Falta configurar conector para tipo {doc_type_name} en empresa {company.short_name}")
-            raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
-                               f"Falta configurar conector para tipo {doc_type_name} en empresa {company.short_name}")
-
-        # get the metadata for this connector
-        predefined_metadata = type_config.get('metadata', {})
-
-        # config specific filters
-        filters = type_config.get('filters', {"filename_contains": ".pdf"})
-
-        return self.load_data_source(connector, predefined_metadata, filters)
-
-    def load_data_source(self, connector_config: Dict, predefined_metadata: Dict = None, filters: Dict = None):
-        """
-        Carga archivos desde una fuente de datos usando un conector.
-
-        Args:
-            connector_config: Configuración del conector
-            predefined_metadata: Metadatos predefinidos para todos los documentos de esta fuente
-            filters: Filtros específicos para esta carga
-
-        Returns:
-            int o dict: Número de archivos procesados o diccionario de error
-        """
-        try:
-            # Si no se proporcionaron filtros, usar el predeterminado
-            if not filters:
-                filters = {"filename_contains": ".pdf"}
-
-            # Pasar metadata predefinida como parte del contexto al procesador
-            # para que esté disponible en la función load_file
-            extra_context = {}
-            if predefined_metadata:
-                extra_context['metadata'] = predefined_metadata
-
-            # config the processor
-            processor_config = FileProcessorConfig(
-                context=extra_context,
-                filters=filters,
-                action=self.load_file,
-                continue_on_error=True,
-                echo=True
-            )
-
-            connector = self.file_connector_factory.create(connector_config)
-            processor = FileProcessor(connector, processor_config)
-
-            # process the files
-            processor.process_files()
-
-            return processor.processed_files
-        except Exception as e:
-            logging.exception("Loading files error: %s", str(e))
-            return {"error": str(e)}
-
-    # load an individual filename
-    # this method is set up on the FileProcessorConfig object
-    def load_file(self, filename: str, content: bytes, context: dict = {}, company: Company = None):
-        if not company:
-            company = self.company
-
-        # check if file exist in repositories
-        if self.doc_repo.get(company=company,filename=filename):
-            return
-
-        try:
-            # extract text from the document
-            document_content = self.doc_service.file_to_txt(filename, content)
-            content_base64 = base64.b64encode(content).decode('utf-8')
-
-            # generate metada based on the filename structure
-            dynamic_metadata = self.dispatcher.get_metadata_from_filename(company_name=company.short_name, filename=filename)
-
-            # Obtener metadatos del contexto si existen
-            context_metadata = context.get('metadata', {}).copy() if context else {}
-
-            # Fusionar los metadatos. El orden de prioridad es:
-            # 1. dynamic_metadata (tiene mayor prioridad)
-            # 2. context_metadata (del parámetro context)
-            # Los valores en dynamic_metadata tendrán precedencia sobre los de context_metadata
-            final_meta = {**context_metadata, **dynamic_metadata}
-
-            # save the file in the document repositories
-            new_document = Document(
-                company_id=company.id,
-                filename=filename,
-                content=document_content,
-                content_b64=content_base64,
-                meta=final_meta
-            )
-
-            # insert the document into the Database (without commit)
-            session = self.doc_repo.session
-            session.add(new_document)
-            session.flush()     # get the ID without commit
-
-            # split the content, and create the chunk list
-            splitted_content = self.splitter.split_text(document_content)
-            chunk_list = [
-                VSDoc(
-                    company_id=company.id,
-                    document_id=new_document.id,
-                    text=text
-                )
-                for text in splitted_content
-            ]
-
-            # save to vector store
-            self.vector_store.add_document(chunk_list)
-
-            # confirm the transaction
-            session.commit()
-
-            return new_document
-        except Exception as e:
-            self.doc_repo.session.rollback()
-
-            # if something fails, throw exception
-            logging.exception("Error procesando el archivo %s: %s", filename, str(e))
-            raise IAToolkitException(IAToolkitException.ErrorType.LOAD_DOCUMENT_ERROR,
-                               f"Error al procesar el archivo {filename}")

services/mail_service.py

@@ -1,62 +0,0 @@
-# Copyright (c) 2024 Fernando Libedinsky
-# Producto: IAToolkit
-# Todos los derechos reservados.
-# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
-
-from infra.mail_app import MailApp
-from injector import inject
-from pathlib import Path
-from common.exceptions import IAToolkitException
-import base64
-
-TEMP_DIR = Path("static/temp")
-
-class MailService:
-    @inject
-    def __init__(self, mail_app: MailApp):
-        self.mail_app = mail_app
-
-    def _read_token_bytes(self, token: str) -> bytes:
-        # Defensa simple contra path traversal
-        if not token or "/" in token or "\\" in token or token.startswith("."):
-            raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
-                               "attachment_token inválido")
-        path = TEMP_DIR / token
-        if not path.is_file():
-            raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
-                               f"Adjunto no encontrado: {token}")
-        return path.read_bytes()
-
-    def send_mail(self, **kwargs):
-        from_email = kwargs.get('from_email', 'iatoolkit@iatoolkit.com')
-        recipient = kwargs.get('recipient')
-        subject = kwargs.get('subject')
-        body = kwargs.get('body')
-        attachments = kwargs.get('attachments')
-
-        # Normalizar a payload de MailApp (name + base64 content)
-        norm_attachments = []
-        for a in attachments or []:
-            if a.get("attachment_token"):
-                raw = self._read_token_bytes(a["attachment_token"])
-                norm_attachments.append({
-                    "filename": a["filename"],
-                    "content": base64.b64encode(raw).decode("utf-8"),
-                })
-            else:
-                # asumo que ya viene un base64
-                norm_attachments.append({
-                    "filename": a["filename"],
-                    "content": a["content"]
-                })
-
-        self.sender = {"email": from_email, "name": "IAToolkit"}
-
-        response = self.mail_app.send_email(
-            sender=self.sender,
-            to=recipient,
-            subject=subject,
-            body=body,
-            attachments=norm_attachments)
-
-        return 'mail enviado'

services/prompt_manager_service.py

@@ -1,172 +0,0 @@
-# Copyright (c) 2024 Fernando Libedinsky
-# Producto: IAToolkit
-# Todos los derechos reservados.
-# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
-
-from injector import inject
-from repositories.llm_query_repo import LLMQueryRepo
-import logging
-from repositories.profile_repo import ProfileRepo
-from collections import defaultdict
-from repositories.models import Prompt, PromptCategory, Company
-import os
-from common.exceptions import IAToolkitException
-from pathlib import Path
-import importlib.resources
-
-
-class PromptService:
-    @inject
-    def __init__(self, llm_query_repo: LLMQueryRepo, profile_repo: ProfileRepo):
-        self.llm_query_repo = llm_query_repo
-        self.profile_repo = profile_repo
-
-    def create_prompt(self,
-                      prompt_name: str,
-                      description: str,
-                      order: int,
-                      company: Company = None,
-                      category: PromptCategory = None,
-                      active: bool = True,
-                      is_system_prompt: bool = False,
-                      params: dict = {}
-                      ):
-
-        prompt_filename = prompt_name.lower() + '.prompt'
-        if is_system_prompt:
-            if not importlib.resources.files('iatoolkit.system_prompts').joinpath(prompt_filename).is_file():
-                raise IAToolkitException(IAToolkitException.ErrorType.INVALID_NAME,
-                                f'No existe el archivo de prompt de sistemas: {prompt_filename}')
-        else:
-            template_dir = f'companies/{company.short_name}/prompts'
-
-            relative_prompt_path = os.path.join(template_dir, prompt_filename)
-            if not os.path.exists(relative_prompt_path):
-                raise IAToolkitException(IAToolkitException.ErrorType.INVALID_NAME,
-                               f'No existe el archivo de prompt: {relative_prompt_path}')
-
-        prompt = Prompt(
-                company_id=company.id if company else None,
-                name=prompt_name,
-                description=description,
-                order=order,
-                category_id=category.id if category and not is_system_prompt else None,
-                active=active,
-                filename=prompt_filename,
-                is_system_prompt=is_system_prompt,
-                parameters=params
-            )
-
-        try:
-            self.llm_query_repo.create_or_update_prompt(prompt)
-        except Exception as e:
-            raise IAToolkitException(IAToolkitException.ErrorType.DATABASE_ERROR,
-                               f'error creating prompt "{prompt_name}": {str(e)}')
-
-    def get_prompt_content(self, company: Company, prompt_name: str):
-        try:
-            user_prompt_content = []
-            execution_dir = os.getcwd()
-
-            # get the user prompt
-            user_prompt = self.llm_query_repo.get_prompt_by_name(company, prompt_name)
-            if not user_prompt:
-                raise IAToolkitException(IAToolkitException.ErrorType.DOCUMENT_NOT_FOUND,
-                                   f"No se encontró el prompt '{prompt_name}' para la empresa '{company.short_name}'")
-
-            absolute_filepath = os.path.join(execution_dir, user_prompt.filename)
-            if not os.path.exists(absolute_filepath):
-                raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                                   f"El archivo para el prompt '{prompt_name}' no existe: {absolute_filepath}")
-
-            try:
-                with open(absolute_filepath, 'r', encoding='utf-8') as f:
-                    user_prompt_content = f.read()
-            except Exception as e:
-                raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                                   f"Error leyendo el archivo de prompt '{prompt_name}' en {absolute_filepath}: {e}")
-
-            return user_prompt_content
-
-        except IAToolkitException:
-            # Vuelve a lanzar las IAToolkitException que ya hemos manejado
-            # para que no sean capturadas por el siguiente bloque.
-            raise
-        except Exception as e:
-            logging.exception(
-                f"Error al obtener el contenido del prompt para la empresa '{company.short_name}' y prompt '{prompt_name}': {e}")
-            raise IAToolkitException(IAToolkitException.ErrorType.PROMPT_ERROR,
-                               f'Error al obtener el contenido del prompt "{prompt_name}" para la empresa {company.short_name}: {str(e)}')
-
-    def get_system_prompt(self):
-        try:
-            system_prompt_content = []
-
-            # read all the system prompts from the database
-            system_prompts = self.llm_query_repo.get_system_prompts()
-
-            for prompt in system_prompts:
-                try:
-                    content = importlib.resources.read_text('iatoolkit.system_prompts', prompt.filename)
-                    system_prompt_content.append(content)
-                except FileNotFoundError:
-                    logging.warning(f"El archivo para el prompt de sistema no existe en el paquete: {prompt.filename}")
-                except Exception as e:
-                    raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                                             f"Error leyendo el archivo de prompt del sistema '{prompt.filename}': {e}")
-
-            # join the system prompts into a single string
-            return "\n".join(system_prompt_content)
-
-        except IAToolkitException:
-            raise
-        except Exception as e:
-            logging.exception(
-                f"Error al obtener el contenido del prompt de sistema: {e}")
-            raise IAToolkitException(IAToolkitException.ErrorType.PROMPT_ERROR,
-                               f'Error al obtener el contenido de los prompts de sistema": {str(e)}')
-
-    def get_user_prompts(self, company_short_name: str) -> dict:
-        try:
-            # validate company
-            company = self.profile_repo.get_company_by_short_name(company_short_name)
-            if not company:
-                return {'error': f'No existe la empresa: {company_short_name}'}
-
-            # get all the prompts
-            all_prompts = self.llm_query_repo.get_prompts(company)
-
-            # Agrupar prompts por categoría
-            prompts_by_category = defaultdict(list)
-            for prompt in all_prompts:
-                if prompt.active:
-                    if prompt.category:
-                        cat_key = (prompt.category.order, prompt.category.name)
-                        prompts_by_category[cat_key].append(prompt)
-
-            # Ordenar los prompts dentro de cada categoría
-            for cat_key in prompts_by_category:
-                prompts_by_category[cat_key].sort(key=lambda p: p.order)
-
-            # Crear la estructura de respuesta final, ordenada por la categoría
-            categorized_prompts = []
-
-            # Ordenar las categorías por su 'order'
-            sorted_categories = sorted(prompts_by_category.items(), key=lambda item: item[0][0])
-
-            for (cat_order, cat_name), prompts in sorted_categories:
-                categorized_prompts.append({
-                    'category_name': cat_name,
-                    'category_order': cat_order,
-                    'prompts': [
-                        {'prompt': p.name, 'description': p.description, 'order': p.order}
-                        for p in prompts
-                    ]
-                })
-
-            return {'message': categorized_prompts}
-
-        except Exception as e:
-            logging.error(f"Error en get_prompts: {e}")
-            return {'error': str(e)}
-