iatoolkit 0.71.4__py3-none-any.whl → 1.4.2__py3-none-any.whl

iatoolkit/services/license_service.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+import jwt
+import os
+import logging
+from pathlib import Path
+from iatoolkit.common.exceptions import IAToolkitException
+from injector import inject, singleton
+
+
+@singleton
+class LicenseService:
+    """
+    Manages system restrictions and features based on a license (JWT).
+    If no license or an invalid license is provided, Community Edition limits apply.
+    """
+    @inject
+    def __init__(self):
+        self.limits = self._load_limits()
+
+    def _load_limits(self):
+        # 1. Define default limits (Community Edition)
+        default_limits = {
+            "license_type": "Community Edition",
+            "plan": "Open Source (Community Edition)",
+            "max_companies": 1,
+            "max_tools": 3,
+            "features": {
+                "multi_tenant": False,
+                "rag_advanced": False,
+            }
+        }
+        return default_limits
+
+
+    # --- Information Getters ---
+    def get_license_type(self) -> str:
+        return self.limits.get("license_type", "Community Edition")
+
+    def get_plan_name(self) -> str:
+        return self.limits.get("plan", "Unknown")
+
+    def get_max_companies(self) -> int:
+        return self.limits.get("max_companies", 1)
+
+    def get_max_tools_per_company(self) -> int:
+        return self.limits.get("max_tools", 3)
+
+    def get_license_info(self) -> str:
+        return f"Plan: {self.get_plan_name()}, Companies: {self.get_max_companies()}, Tools: {self.get_max_tools_per_company()}"
+
+    # --- Restriction Validators ---
+
+    def validate_company_limit(self, current_count: int):
+        """Raises exception if the limit of active companies is exceeded."""
+        limit = self.get_max_companies()
+        # -1 means unlimited
+        if limit != -1 and current_count > limit:
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.PERMISSION,
+                f"Company limit ({limit}) reached for plan '{self.get_plan_name()}'."
+            )
+
+
+    def validate_tool_config_limit(self, tools_config: list):
+        """Validates a configuration list before processing it."""
+        limit = self.get_max_tools_per_company()
+        if limit != -1 and len(tools_config) > limit:
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.PERMISSION,
+                f"Configuration defines {len(tools_config)} tools, but limit is {limit}."
+            )
+
+    # --- Feature Gating Validators ---
+
+    def has_feature(self, feature_key: str) -> bool:
+        """Checks if a specific feature is enabled in the license."""
+        features = self.limits.get("features", {})
+        return features.get(feature_key, False)

iatoolkit/{infra/llm_client.py → services/llm_client_service.py}

@@ -8,6 +8,7 @@ from iatoolkit.repositories.models import Company, LLMQuery
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
 from sqlalchemy.exc import SQLAlchemyError, OperationalError
 from iatoolkit.common.util import Utility
+from iatoolkit.common.model_registry import ModelRegistry
 from injector import inject
 import time
 import markdown2
@@ -30,11 +31,13 @@ class llmClient:
     @inject
     def __init__(self,
                  llmquery_repo: LLMQueryRepo,
-                 llm_proxy_factory: LLMProxy,
+                 llm_proxy: LLMProxy,
+                 model_registry: ModelRegistry,
                  util: Utility
                  ):
         self.llmquery_repo = llmquery_repo
-        self.llm_proxy_factory = llm_proxy_factory
+        self.llm_proxy = llm_proxy
+        self.model_registry = model_registry
         self.util = util
         self._dispatcher = None # Cache for the lazy-loaded dispatcher
 
@@ -73,33 +76,31 @@ class llmClient:
         response = None
         sql_retry_count = 0
         force_tool_name = None
-        reasoning = {}
 
-        if 'gpt-5' in model:
-            text['verbosity'] = "low"
-            reasoning = {"effort": 'minimal'}
+        # Resolve per-model defaults and apply overrides (without mutating inputs).
+        request_params = self.model_registry.resolve_request_params(model=model, text=text)
+        text_payload = request_params["text"]
+        reasoning = request_params["reasoning"]
 
         try:
             start_time = time.time()
-            logging.info(f"calling llm model '{model}' with {self.count_tokens(context)} tokens...")
+            logging.info(f"calling llm model '{model}' with {self.count_tokens(context, context_history)} tokens...")
 
-            # get the proxy for the company
-            llm_proxy = self.llm_proxy_factory.create_for_company(company)
-
-            # here is the first call to the LLM
+            # this is the first call to the LLM on the iteration
             try:
                 input_messages = [{
                     "role": "user",
                     "content": context
                 }]
 
-                response = llm_proxy.create_response(
+                response = self.llm_proxy.create_response(
+                    company_short_name=company.short_name,
                     model=model,
+                    input=input_messages,
                     previous_response_id=previous_response_id,
                     context_history=context_history,
-                    input=input_messages,
                     tools=tools,
-                    text=text,
+                    text=text_payload,
                     reasoning=reasoning,
                 )
                 stats = self.get_stats(response)
@@ -127,12 +128,18 @@ class llmClient:
                     # execute the function call through the dispatcher
                     fcall_time = time.time()
                     function_name = tool_call.name
-                    args = json.loads(tool_call.arguments)
-                    logging.info(f"start execution fcall: {function_name}")
+
+                    try:
+                        args = json.loads(tool_call.arguments)
+                    except Exception as e:
+                        logging.error(f"[Dispatcher] json.loads failed: {e}")
+                        raise
+                    logging.debug(f"[Dispatcher] Parsed args = {args}")
+
                     try:
                         result = self.dispatcher.dispatch(
                             company_short_name=company.short_name,
-                            action=function_name,
+                            function_name=function_name,
                             **args
                         )
                         force_tool_name = None
@@ -160,6 +167,7 @@ class llmClient:
                     input_messages.append({
                         "type": "function_call_output",
                         "call_id": tool_call.call_id,
+                        "status": "completed",
                         "output": str(result)
                     })
                     function_calls = True
@@ -170,17 +178,18 @@ class llmClient:
                     f_calls.append(f_call_identity)
                     f_call_time += elapsed
 
-                    logging.info(f"end execution {function_name} in {elapsed:.1f} secs.")
+                    logging.info(f"[{company.short_name}] end execution of tool: {function_name} in {elapsed:.1f} secs.")
 
                 if not function_calls:
-                    break           # no function call answer to send back to llm
+                    break           # no more function calls, the answer to send back to llm
 
                 # send results back to the LLM
                 tool_choice_value = "auto"
                 if force_tool_name:
                     tool_choice_value = "required"
 
-                response = llm_proxy.create_response(
+                response = self.llm_proxy.create_response(
+                    company_short_name=company.short_name,
                     model=model,
                     input=input_messages,
                     previous_response_id=response.id,
@@ -188,7 +197,7 @@ class llmClient:
                     reasoning=reasoning,
                     tool_choice=tool_choice_value,
                     tools=tools,
-                    text=text
+                    text=text_payload,
                 )
                 stats_fcall = self.add_stats(stats_fcall, self.get_stats(response))
 
@@ -200,9 +209,11 @@ class llmClient:
             # decode the LLM response
             decoded_response = self.decode_response(response)
 
+            # Extract reasoning from the final response object
+            final_reasoning = getattr(response, 'reasoning_content', '')
+
             # save the query and response
             query = LLMQuery(user_identifier=user_identifier,
-                             task_id=0,
                              company_id=company.id,
                              query=question,
                              output=decoded_response.get('answer', ''),
@@ -226,6 +237,8 @@ class llmClient:
                 'aditional_data': decoded_response.get('aditional_data', {}),
                 'response_id': response.id,
                 'query_id': query.id,
+                'model': model,
+                'reasoning_content': final_reasoning,
             }
         except SQLAlchemyError as db_error:
             # rollback
@@ -240,11 +253,10 @@ class llmClient:
 
             # log the error in the llm_query table
             query = LLMQuery(user_identifier=user_identifier,
-                             task_id=0,
                              company_id=company.id,
                              query=question,
                              output=error_message,
-                             response=response.output_text if response else {},
+                             response={},
                              valid_response=False,
                              function_calls=f_calls,
                              )
@@ -265,14 +277,15 @@ class llmClient:
 
         logging.info(f"initializing model '{model}' with company context: {self.count_tokens(company_base_context)} tokens...")
 
-        llm_proxy = self.llm_proxy_factory.create_for_company(company)
         try:
-            response = llm_proxy.create_response(
+            response = self.llm_proxy.create_response(
+                company_short_name=company.short_name,
                 model=model,
                 input=[{
                     "role": "system",
                     "content": company_base_context
-                }]
+                }],
+
             )
 
         except Exception as e:
@@ -419,7 +432,7 @@ class llmClient:
         html_answer = markdown2.markdown(answer).replace("\n", "")
         return html_answer
 
-    def count_tokens(self, text):
+    def count_tokens(self, text, history = []):
         # Codifica el texto y cuenta la cantidad de tokens
-        tokens = self.encoding.encode(text)
+        tokens = self.encoding.encode(text + json.dumps(history))
         return len(tokens)

iatoolkit/services/load_documents_service.py

@@ -1,17 +1,13 @@
 # Copyright (c) 2024 Fernando Libedinsky
 # Product: IAToolkit
 
-from iatoolkit.repositories.vs_repo import VSRepo
-from iatoolkit.repositories.document_repo import DocumentRepo
-from iatoolkit.repositories.models import Document, VSDoc, Company
-from iatoolkit.services.document_service import DocumentService
+from iatoolkit.repositories.models import Company
 from iatoolkit.services.configuration_service import ConfigurationService
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from iatoolkit.services.knowledge_base_service import KnowledgeBaseService
 from iatoolkit.infra.connectors.file_connector_factory import FileConnectorFactory
 from iatoolkit.services.file_processor_service import FileProcessorConfig, FileProcessor
 from iatoolkit.common.exceptions import IAToolkitException
 import logging
-import base64
 from injector import inject, singleton
 import os
 
@@ -19,31 +15,21 @@ import os
 @singleton
 class LoadDocumentsService:
     """
-    Orchestrates the process of loading, processing, and storing documents
-    from various sources defined in the company's configuration.
+    Orchestrates the discovery and loading of documents from configured sources.
+    Delegates the processing and ingestion logic to KnowledgeBaseService.
     """
     @inject
     def __init__(self,
                  config_service: ConfigurationService,
                  file_connector_factory: FileConnectorFactory,
-                 doc_service: DocumentService,
-                 doc_repo: DocumentRepo,
-                 vector_store: VSRepo,
+                 knowledge_base_service: KnowledgeBaseService
                  ):
         self.config_service = config_service
-        self.doc_service = doc_service
-        self.doc_repo = doc_repo
-        self.vector_store = vector_store
         self.file_connector_factory = file_connector_factory
+        self.knowledge_base_service = knowledge_base_service
 
         logging.getLogger().setLevel(logging.ERROR)
 
-        self.splitter = RecursiveCharacterTextSplitter(
-            chunk_size=1000,
-            chunk_overlap=100,
-            separators=["\n\n", "\n", "."]
-        )
-
     def load_sources(self,
                      company: Company,
                      sources_to_load: list[str] = None,
@@ -67,7 +53,7 @@ class LoadDocumentsService:
 
         if not sources_to_load:
             raise IAToolkitException(IAToolkitException.ErrorType.PARAM_NOT_FILLED,
-                                f"Missing sources to load for company '{company.short_name}'.")
+                                     f"Missing sources to load for company '{company.short_name}'.")
 
         base_connector_config = self._get_base_connector_config(knowledge_base_config)
         all_sources = knowledge_base_config.get('document_sources', {})
@@ -89,6 +75,7 @@ class LoadDocumentsService:
                 # Prepare the context for the callback function.
                 context = {
                     'company': company,
+                    'collection': source_config.get('metadata', {}).get('collection'),
                     'metadata': source_config.get('metadata', {})
                 }
 
@@ -130,45 +117,29 @@ class LoadDocumentsService:
 
     def _file_processing_callback(self, company: Company, filename: str, content: bytes, context: dict = None):
         """
-        Callback method to process a single file. It extracts text, merges metadata,
-        and saves the document to both relational and vector stores.
+        Callback method to process a single file.
+        Delegates the actual ingestion (storage, vectorization) to KnowledgeBaseService.
         """
         if not company:
             raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER, "Missing company object in callback.")
 
-        if self.doc_repo.get(company_id=company.id, filename=filename):
-            logging.debug(f"File '{filename}' already exists for company '{company.id}'. Skipping.")
-            return
-
         try:
-            document_content = self.doc_service.file_to_txt(filename, content)
-
             # Get predefined metadata from the context passed by the processor.
             predefined_metadata = context.get('metadata', {}) if context else {}
 
-            # Save the document to the relational database.
-            session = self.doc_repo.session
-            new_document = Document(
-                company_id=company.id,
+            # Delegate heavy lifting to KnowledgeBaseService
+            new_document = self.knowledge_base_service.ingest_document_sync(
+                company=company,
                 filename=filename,
-                content=document_content,
-                content_b64=base64.b64encode(content).decode('utf-8'),
-                meta=predefined_metadata
+                content=content,
+                collection=predefined_metadata.get('collection'),
+                metadata=predefined_metadata
             )
-            session.add(new_document)
-            session.flush()  # Flush to get the new_document.id without committing.
 
-            # Split into chunks and prepare for vector store.
-            chunks = self.splitter.split_text(document_content)
-            vs_docs = [VSDoc(company_id=company.id, document_id=new_document.id, text=text) for text in chunks]
-
-            # Add document chunks to the vector store.
-            self.vector_store.add_document(company.short_name, vs_docs)
-
-            session.commit()
             return new_document
+
         except Exception as e:
-            self.doc_repo.session.rollback()
+            # We log here but re-raise to let FileProcessor handle the error counting/continue logic
             logging.exception(f"Error processing file '{filename}': {e}")
             raise IAToolkitException(IAToolkitException.ErrorType.LOAD_DOCUMENT_ERROR,
                                      f"Error while processing file: {filename}")

iatoolkit/services/mail_service.py

@@ -3,43 +3,40 @@
 #
 # IAToolkit is open source software.
 
-from iatoolkit.infra.mail_app import MailApp
+from iatoolkit.services.configuration_service import ConfigurationService
 from iatoolkit.services.i18n_service import I18nService
+from iatoolkit.infra.brevo_mail_app import BrevoMailApp
 from injector import inject
 from pathlib import Path
-from iatoolkit.common.exceptions import IAToolkitException
 import base64
+import os
+import smtplib
+from email.message import EmailMessage
+from iatoolkit.common.exceptions import IAToolkitException
+
 
 TEMP_DIR = Path("static/temp")
 
 class MailService:
     @inject
     def __init__(self,
-                 mail_app: MailApp,
-                 i18n_service: I18nService):
+                 config_service: ConfigurationService,
+                 mail_app: BrevoMailApp,
+                 i18n_service: I18nService,
+                 brevo_mail_app: BrevoMailApp):
         self.mail_app = mail_app
+        self.config_service = config_service
         self.i18n_service = i18n_service
+        self.brevo_mail_app = brevo_mail_app
 
 
-    def _read_token_bytes(self, token: str) -> bytes:
-        # Defensa simple contra path traversal
-        if not token or "/" in token or "\\" in token or token.startswith("."):
-            raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
-                               "attachment_token invalid")
-        path = TEMP_DIR / token
-        if not path.is_file():
-            raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
-                               f"attach file not found: {token}")
-        return path.read_bytes()
-
-    def send_mail(self, **kwargs):
-        from_email = kwargs.get('from_email', 'iatoolkit@iatoolkit.com')
+    def send_mail(self, company_short_name: str, **kwargs):
         recipient = kwargs.get('recipient')
         subject = kwargs.get('subject')
         body = kwargs.get('body')
         attachments = kwargs.get('attachments')
 
-        # Normalizar a payload de MailApp (name + base64 content)
+        # Normalizar a payload de BrevoMailApp (name + base64 content)
         norm_attachments = []
         for a in attachments or []:
             if a.get("attachment_token"):
@@ -55,13 +52,162 @@ class MailService:
                     "content": a["content"]
                 })
 
-        self.sender = {"email": from_email, "name": "IAToolkit"}
+        # build provider configuration from company.yaml
+        provider, provider_config = self._build_provider_config(company_short_name)
+
+        # define the email sender
+        sender = {
+            "email": provider_config.get("sender_email"),
+            "name": provider_config.get("sender_name"),
+        }
 
-        response = self.mail_app.send_email(
-            sender=self.sender,
-            to=recipient,
-            subject=subject,
-            body=body,
-            attachments=norm_attachments)
+        # select provider and send the email through it
+        if provider == "brevo_mail":
+            response = self.brevo_mail_app.send_email(
+                provider_config=provider_config,
+                sender=sender,
+                to=recipient,
+                subject=subject,
+                body=body,
+                attachments=norm_attachments
+            )
+        elif provider == "smtplib":
+            response = self._send_with_smtplib(
+                provider_config=provider_config,
+                sender=sender,
+                recipient=recipient,
+                subject=subject,
+                body=body,
+                attachments=norm_attachments,
+            )
+            response = None
+        else:
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.MAIL_ERROR,
+                f"Unknown mail provider '{provider}'"
+            )
 
         return self.i18n_service.t('services.mail_sent')
+
+    def _build_provider_config(self, company_short_name: str) -> tuple[str, dict]:
+        """
+        Determina el provider activo (brevo_mail / smtplib) y construye
+        el diccionario de configuración a partir de las variables de entorno
+        cuyos nombres están en company.yaml (mail_provider).
+        """
+        # get company mail configuration and provider
+        mail_config = self.config_service.get_configuration(company_short_name, "mail_provider")
+        provider = mail_config.get("provider", "brevo_mail")
+
+        # get mail common parameteres
+        sender_email = mail_config.get("sender_email")
+        sender_name = mail_config.get("sender_name")
+
+        # get parameters depending on provider
+        if provider == "brevo_mail":
+            brevo_cfg = mail_config.get("brevo_mail", {})
+            api_key_env = brevo_cfg.get("brevo_api", "BREVO_API_KEY")
+            return provider, {
+                "api_key": os.getenv(api_key_env),
+                "sender_name": sender_name,
+                "sender_email": sender_email,
+            }
+
+        if provider == "smtplib":
+            smtp_cfg = mail_config.get("smtplib", {})
+            host = os.getenv(smtp_cfg.get("host_env", "SMTP_HOST"))
+            port = os.getenv(smtp_cfg.get("port_env", "SMTP_PORT"))
+            username = os.getenv(smtp_cfg.get("username_env", "SMTP_USERNAME"))
+            password = os.getenv(smtp_cfg.get("password_env", "SMTP_PASSWORD"))
+            use_tls = os.getenv(smtp_cfg.get("use_tls_env", "SMTP_USE_TLS"))
+            use_ssl = os.getenv(smtp_cfg.get("use_ssl_env", "SMTP_USE_SSL"))
+
+            return provider, {
+                "host": host,
+                "port": int(port) if port is not None else None,
+                "username": username,
+                "password": password,
+                "use_tls": str(use_tls).lower() == "true",
+                "use_ssl": str(use_ssl).lower() == "true",
+                "sender_name": sender_name,
+                "sender_email": sender_email,
+            }
+
+        # Fallback simple si el provider no es reconocido
+        raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
+                                 f"missing mail provider in mail configuration for company '{company_short_name}'")
+
+    def _send_with_smtplib(self,
+                           provider_config: dict,
+                           sender: dict,
+                           recipient: str,
+                           subject: str,
+                           body: str,
+                           attachments: list[dict] | None):
+        """
+        Envía correo usando smtplib, utilizando la configuración normalizada
+        en provider_config.
+        """
+        host = provider_config.get("host")
+        port = provider_config.get("port")
+        username = provider_config.get("username")
+        password = provider_config.get("password")
+        use_tls = provider_config.get("use_tls")
+        use_ssl = provider_config.get("use_ssl")
+
+        if not host or not port:
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.MAIL_ERROR,
+                "smtplib configuration is incomplete (host/port missing)"
+            )
+
+        msg = EmailMessage()
+        msg["From"] = f"{sender.get('name', '')} <{sender.get('email')}>"
+        msg["To"] = recipient
+        msg["Subject"] = subject
+        msg.set_content(body, subtype="html")
+
+        # Adjuntos: ya vienen como filename + base64 content
+        for a in attachments or []:
+            filename = a.get("filename")
+            content_b64 = a.get("content")
+            if not filename or not content_b64:
+                continue
+            try:
+                raw = base64.b64decode(content_b64, validate=True)
+            except Exception:
+                raise IAToolkitException(
+                    IAToolkitException.ErrorType.MAIL_ERROR,
+                    f"Invalid base64 for attachment '{filename}'"
+                )
+            msg.add_attachment(
+                raw,
+                maintype="application",
+                subtype="octet-stream",
+                filename=filename,
+            )
+
+        if use_ssl:
+            with smtplib.SMTP_SSL(host, port) as server:
+                if username and password:
+                    server.login(username, password)
+                server.send_message(msg)
+        else:
+            with smtplib.SMTP(host, port) as server:
+                if use_tls:
+                    server.starttls()
+                if username and password:
+                    server.login(username, password)
+                server.send_message(msg)
+
+
+    def _read_token_bytes(self, token: str) -> bytes:
+        # Defensa simple contra path traversal
+        if not token or "/" in token or "\\" in token or token.startswith("."):
+            raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
+                               "attachment_token invalid")
+        path = TEMP_DIR / token
+        if not path.is_file():
+            raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
+                               f"attach file not found: {token}")
+        return path.read_bytes()