iatoolkit 1.10.0__tar.gz → 1.21.0__tar.gz

{iatoolkit-1.10.0/src/iatoolkit.egg-info → iatoolkit-1.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 1.10.0
+Version: 1.21.0
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "1.10.0"
+__version__ = "1.21.0"
 
 # Expose main classes and functions at the top level of the package
 
@@ -19,7 +19,7 @@ from iatoolkit.services.query_service import QueryService
 from iatoolkit.services.document_service import DocumentService
 from iatoolkit.services.knowledge_base_service import KnowledgeBaseService
 from iatoolkit.services.sql_service import SqlService
-from iatoolkit.services.load_documents_service import LoadDocumentsService
+from iatoolkit.services.ingestor_service import IngestorService
 from iatoolkit.infra.call_service import CallServiceClient
 from iatoolkit.services.profile_service import ProfileService
 from iatoolkit.services.mail_service import MailService
@@ -37,7 +37,7 @@ __all__ = [
     'SqlService',
     'DocumentService',
     'KnowledgeBaseService',
-    'LoadDocumentsService',
+    'IngestorService',
     'CallServiceClient',
     'ProfileService',
     'MailService',

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/cli_commands.py

@@ -7,6 +7,7 @@ import click
 import logging
 from iatoolkit.core import IAToolkit
 from iatoolkit.services.profile_service import ProfileService
+from iatoolkit.services.prompt_service import PromptService
 
 
 def register_core_commands(app):
@@ -32,6 +33,21 @@ def register_core_commands(app):
             logging.exception(e)
             click.echo(f"❌ unexpectd error during the configuration: {e}")
 
+    @app.cli.command("init-company")
+    @click.argument("company_short_name")
+    def init_company(company_short_name: str):
+        """⚙️ Bootstrap a new company."""
+        try:
+            prompt_service = IAToolkit.get_instance().get_injector().get(PromptService)
+            click.echo(f"🔑 Bootstrap company: '{company_short_name}'...")
+            result = prompt_service.register_system_prompts(company_short_name)
+
+            if result:
+                click.echo("✅ System prompts registered successfully!")
+        except Exception as e:
+            logging.exception(e)
+            click.echo(f"❌ unexpected error during the configuration: {e}")
+
     @app.cli.command("encrypt-key")
     @click.argument("key")
     def encrypt_llm_api_key(key: str):

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/common/routes.py

@@ -33,6 +33,7 @@ def register_views(app):
     from iatoolkit.views.users_api_view import UsersApiView
     from iatoolkit.views.rag_api_view import RagApiView
     from iatoolkit.views.categories_api_view import CategoriesApiView
+    from iatoolkit.views.ingestion_api_view import IngestionApiView
 
     # assign root '/' to our new redirect logic
     app.add_url_rule('/home', view_func=RootRedirectView.as_view('root_redirect'))
@@ -136,6 +137,11 @@ def register_views(app):
     # this endpoint is for upload documents into the vector store (api-key)
     app.add_url_rule('/api/load-document', view_func=LoadDocumentApiView.as_view('load-document'), methods=['POST'])
 
+    # Document ingestion
+    ingestion_view = IngestionApiView.as_view('ingestion_api')
+    app.add_url_rule('/<company_short_name>/api/ingestion-sources', view_func=ingestion_view, methods=['GET', 'POST'])
+    app.add_url_rule('/<company_short_name>/api/ingestion-sources/<int:source_id>/<action>', view_func=ingestion_view, methods=['POST'])
+
     # this endpoint is for generating embeddings for a given text
     app.add_url_rule('/<company_short_name>/api/embedding',
                      view_func=EmbeddingApiView.as_view('embedding_api'))

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/common/util.py

@@ -162,8 +162,15 @@ class Utility:
         Parses a YAML string into a dictionary securely.
         """
         try:
+            if not yaml_content:
+                return {}
+
+            # Normalizar tabulaciones que rompen YAML
             yaml_content = yaml_content.replace('\t', '  ')
-            return yaml.safe_load(yaml_content) or {}
+
+            loaded = yaml.safe_load(yaml_content)
+            # Asegurar que siempre retornamos un dict, incluso si el YAML es una lista o escalar
+            return loaded if isinstance(loaded, dict) else {}
         except yaml.YAMLError as e:
             logging.error(f"Error parsing YAML string: {e}")
             return {}
@@ -182,128 +189,6 @@ class Utility:
             raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
                                      f"Failed to generate YAML: {e}")
 
-    def generate_context_for_schema(self, entity_name: str, schema_file: str = None, schema: dict = {}) -> str:
-        if not schema_file and not schema:
-            raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                               f'No se pudo obtener schema de la entidad: {entity_name}')
-
-        try:
-            if schema_file:
-                schema = self.load_schema_from_yaml(schema_file)
-            table_schema = self.generate_schema_table(schema)
-            return table_schema
-        except Exception as e:
-            logging.exception(e)
-            raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                               f'No se pudo leer el schema de la entidad: {entity_name}') from e
-
-    def generate_schema_table(self, schema: dict) -> str:
-        """
-        Genera una descripción detallada y formateada en Markdown de un esquema.
-        Esta función está diseñada para manejar el formato específico de nuestros
-        archivos YAML, donde el esquema se define bajo una única clave raíz.
-        """
-        if not schema or not isinstance(schema, dict):
-            return ""
-
-        # Asumimos que el YAML tiene una única clave raíz que nombra a la entidad.
-        if len(schema) == 1:
-            root_name = list(schema.keys())[0]
-            root_details = schema[root_name]
-
-            # support this format
-            if root_details.get('columns'):
-                root_details = root_details['columns']
-
-            if isinstance(root_details, dict):
-                # Las claves de metadatos describen el objeto en sí, no sus propiedades hijas.
-                METADATA_KEYS = ['description', 'type', 'format', 'items', 'properties', 'pk']
-
-                # Las propiedades son las claves restantes en el diccionario.
-                properties = {
-                    k: v for k, v in root_details.items() if k not in METADATA_KEYS
-                }
-
-                # La descripción del objeto raíz.
-                root_description = root_details.get('description', '')
-
-                # Formatea las propiedades extraídas usando la función auxiliar recursiva.
-                formatted_properties = self._format_json_schema(properties, 0)
-
-                # Construcción del resultado final, incluyendo el nombre del objeto raíz.
-                output_parts = [f"\n\n### Objeto: `{root_name}`"]
-                if root_description:
-                    # Limpia la descripción para que se muestre bien
-                    cleaned_description = '\n'.join(line.strip() for line in root_description.strip().split('\n'))
-                    output_parts.append(f"{cleaned_description}")
-
-                if formatted_properties:
-                    output_parts.append(f"**Campos del objeto `{root_name}`:**\n{formatted_properties}")
-
-                return "\n".join(output_parts)
-
-        # Si el esquema (como tender_schema.yaml) no tiene un objeto raíz,
-        # se formatea directamente como una lista de propiedades.
-        return self._format_json_schema(schema, 0)
-
-    def _format_json_schema(self, properties: dict, indent_level: int) -> str:
-        """
-        Formatea de manera recursiva las propiedades de un esquema JSON/YAML.
-        """
-        output = []
-        indent_str = '  ' * indent_level
-
-        for name, details in properties.items():
-            if not isinstance(details, dict):
-                continue
-
-            description = details.get('description', '')
-            data_type = details.get('type', 'any')
-            output.append(f"{indent_str}- **`{name.lower()}`** ({data_type}): {description}")
-            # if 'pk' in details and details['pk']:
-            #    output.append(f"{indent_str}- **Primary Key**: {details['pk']}")
-
-            child_indent_str = '  ' * (indent_level + 1)
-
-            # Manejo de 'oneOf' para mostrar valores constantes
-            if 'oneOf' in details:
-                for item in details['oneOf']:
-                    if 'const' in item:
-                        const_desc = item.get('description', '')
-                        output.append(f"{child_indent_str}- `{item['const']}`: {const_desc}")
-
-            # Manejo de 'items' para arrays
-            if 'items' in details:
-                items_details = details.get('items', {})
-                if isinstance(items_details, dict):
-                    item_description = items_details.get('description')
-                    if item_description:
-                        # Limpiamos y añadimos la descripción del item
-                        cleaned_description = '\n'.join(
-                            f"{line.strip()}" for line in item_description.strip().split('\n')
-                        )
-                        output.append(
-                            f"{child_indent_str}*Descripción de los elementos del array:*\n{child_indent_str}{cleaned_description}")
-
-                    if 'properties' in items_details:
-                        nested_properties = self._format_json_schema(items_details['properties'], indent_level + 1)
-                        output.append(nested_properties)
-
-            # Manejo de 'properties' para objetos anidados estándar
-            if 'properties' in details:
-                nested_properties = self._format_json_schema(details['properties'], indent_level + 1)
-                output.append(nested_properties)
-
-            elif 'additionalProperties' in details and 'properties' in details.get('additionalProperties', {}):
-                # Imprime un marcador de posición para la clave dinámica.
-                output.append(
-                    f"{child_indent_str}- **[*]** (object): Las claves de este objeto son dinámicas (ej. un ID).")
-                # Procesa las propiedades del objeto anidado.
-                nested_properties = self._format_json_schema(details['additionalProperties']['properties'],
-                                                             indent_level + 2)
-                output.append(nested_properties)
-
-        return '\n'.join(output)
 
     def load_markdown_context(self, filepath: str) -> str:
         with open(filepath, 'r', encoding='utf-8') as f:

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/core.py

@@ -58,6 +58,7 @@ class IAToolkit:
         self._injector = Injector()         # init empty injector
         self.version = IATOOLKIT_VERSION
         self.license = "Community Edition"
+        self.is_community = True
 
     @classmethod
     def get_instance(cls) -> 'IAToolkit':
@@ -321,7 +322,7 @@ class IAToolkit:
         from iatoolkit.services.prompt_service import PromptService
         from iatoolkit.services.excel_service import ExcelService
         from iatoolkit.services.mail_service import MailService
-        from iatoolkit.services.load_documents_service import LoadDocumentsService
+        from iatoolkit.services.ingestor_service import IngestorService
         from iatoolkit.services.profile_service import ProfileService
         from iatoolkit.services.jwt_service import JWTService
         from iatoolkit.services.dispatcher_service import Dispatcher
@@ -343,7 +344,7 @@ class IAToolkit:
         binder.bind(PromptService, to=PromptService)
         binder.bind(ExcelService, to=ExcelService)
         binder.bind(MailService, to=MailService)
-        binder.bind(LoadDocumentsService, to=LoadDocumentsService)
+        binder.bind(IngestorService, to=IngestorService)
         binder.bind(ProfileService, to=ProfileService)
         binder.bind(JWTService, to=JWTService)
         binder.bind(Dispatcher, to=Dispatcher)

iatoolkit-1.21.0/src/iatoolkit/infra/connectors/file_connector.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+
+class FileConnector(ABC):
+    @abstractmethod
+    def list_files(self) -> List[str]:
+        pass
+
+    @abstractmethod
+    def get_file_content(self, file_path: str) -> bytes:
+        pass
+
+    @abstractmethod
+    def delete_file(self, file_path: str) -> None:
+        pass
+
+    @abstractmethod
+    def upload_file(self, file_path: str, content: bytes, content_type: str = None) -> None:
+        pass
+
+    def generate_presigned_url(self, file_path: str, expiration: int = 3600) -> Optional[str]:
+        return None

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/infra/connectors/file_connector_factory.py

@@ -1,29 +1,19 @@
-# Copyright (c) 2024 Fernando Libedinsky
-# Product: IAToolkit
-#
-# IAToolkit is open source software.
-
+# ... existing code ...
+import os
+from typing import Dict
 from iatoolkit.infra.connectors.file_connector import FileConnector
 from iatoolkit.infra.connectors.local_file_connector import LocalFileConnector
 from iatoolkit.infra.connectors.s3_connector import S3Connector
-from iatoolkit.infra.connectors.google_drive_connector import GoogleDriveConnector
 from iatoolkit.infra.connectors.google_cloud_storage_connector import GoogleCloudStorageConnector
-from typing import Dict
-import os
-
+from iatoolkit.infra.connectors.google_drive_connector import GoogleDriveConnector
 
 class FileConnectorFactory:
     @staticmethod
     def create(config: Dict) -> FileConnector:
         """
-        Configuración esperada:
-        {
-            "type": "local" | "s3" | "gdrive" | "gcs",
-            "path": "/ruta/local",  # solo para local
-            "bucket": "mi-bucket", "prefix": "datos/", "auth": {...},  # solo para S3
-            "folder_id": "xxxxxxx",  # solo para Google Drive
-            "bucket": "mi-bucket", "service_account": "/ruta/service_account.json"  # solo para GCS
-        }
+        Crea un conector basado en un diccionario de configuración.
+        Permite pasar credenciales explícitas en 'auth' o 'service_account_path',
+        o dejar que el conector use sus defaults.
         """
         connector_type = config.get('type')
 
@@ -31,11 +21,14 @@ class FileConnectorFactory:
             return LocalFileConnector(config['path'])
 
         elif connector_type == 's3':
-            auth = {
-                'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
-                'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
-                'region_name': os.getenv('AWS_REGION', 'us-east-1')
-            }
+            # Permite inyectar auth ya resuelto, o usar defaults de entorno
+            auth = config.get('auth')
+            if not auth:
+                auth = {
+                    'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
+                    'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
+                    'region_name': os.getenv('AWS_REGION', 'us-east-1')
+                }
 
             return S3Connector(
                 bucket=config['bucket'],
@@ -45,14 +38,16 @@ class FileConnectorFactory:
             )
 
         elif connector_type == 'gdrive':
-            return GoogleDriveConnector(config['folder_id'],
-                    'service_account.json')
+            return GoogleDriveConnector(
+                folder_id=config['folder_id'],
+                service_account_path=config.get('service_account', 'service_account.json')
+            )
 
-        elif connector_type == 'gcs':
+        elif connector_type in ['gcs', 'google_cloud_storage']:
             return GoogleCloudStorageConnector(
                 bucket_name=config['bucket'],
-                service_account_path=config.get('service_account', 'service_account.json')
+                service_account_path=config.get('service_account_path', 'service_account.json')
             )
 
         else:
-            raise ValueError(f"Unknown connector type: {connector_type}")
+            raise ValueError(f"Unknown connector type: {connector_type}")

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/infra/connectors/google_cloud_storage_connector.py

@@ -51,3 +51,17 @@ class GoogleCloudStorageConnector(FileConnector):
         file_buffer = blob.download_as_bytes()  # Descarga el contenido como bytes
 
         return file_buffer
+
+    def delete_file(self, file_path: str) -> None:
+        """
+        Elimina un archivo del bucket dado su path.
+        """
+        blob = self.bucket.blob(file_path)
+        blob.delete()
+
+    def upload_file(self, file_path: str, content: bytes, content_type: str = None) -> None:
+        """
+        Sube un archivo al bucket.
+        """
+        blob = self.bucket.blob(file_path)
+        blob.upload_from_string(content, content_type=content_type)

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/infra/connectors/google_drive_connector.py

@@ -5,7 +5,7 @@
 
 from iatoolkit.infra.connectors.file_connector import FileConnector
 from googleapiclient.discovery import build
-from googleapiclient.http import MediaIoBaseDownload
+from googleapiclient.http import MediaIoBaseDownload, MediaIoBaseUpload
 from google.oauth2.service_account import Credentials
 import io
 from typing import List
@@ -66,3 +66,34 @@ class GoogleDriveConnector(FileConnector):
             status, done = downloader.next_chunk()
 
         return file_buffer.getvalue()
+
+    def upload_file(self, file_path: str, content: bytes, content_type: str = None) -> None:
+        """
+        Sube un archivo a Google Drive.
+        Nota: 'file_path' en este contexto se interpreta como el nombre del archivo,
+        ya que GDrive usa IDs para carpetas. El archivo se subirá a la carpeta configurada (self.folder_id).
+        """
+        file_metadata = {
+            'name': file_path, # Usamos file_path como nombre
+            'parents': [self.folder_id]
+        }
+
+        media = MediaIoBaseUpload(io.BytesIO(content), mimetype=content_type, resumable=True)
+
+        self.drive_service.files().create(
+            body=file_metadata,
+            media_body=media,
+            fields='id'
+        ).execute()
+
+    def delete_file(self, file_path: str) -> None:
+        """
+        Elimina un archivo de Google Drive.
+        Nota: 'file_path' aquí DEBE ser el ID del archivo (fileId), no su nombre.
+        """
+        try:
+            self.drive_service.files().delete(fileId=file_path).execute()
+        except Exception:
+            # Si falla (ej: no existe), podríamos loguear o ignorar según diseño.
+            # Aquí asumimos propagación de error o manejo silencioso si no crítico.
+            pass

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/infra/connectors/local_file_connector.py

@@ -44,3 +44,25 @@ class LocalFileConnector(FileConnector):
         except Exception as e:
             raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
                                f"Error leyendo el archivo {file_path}: {e}")
+
+    def upload_file(self, file_path: str, content: bytes, content_type: str = None) -> None:
+        # Nota: file_path debe ser relativo al directorio raíz configurado
+        full_path = os.path.join(self.directory, file_path)
+
+        # Asegurar que el directorio destino existe
+        try:
+            os.makedirs(os.path.dirname(full_path), exist_ok=True)
+            with open(full_path, 'wb') as f:
+                f.write(content)
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
+                                     f"Error escribiendo el archivo {file_path}: {e}")
+
+    def delete_file(self, file_path: str) -> None:
+        full_path = os.path.join(self.directory, file_path)
+        try:
+            if os.path.exists(full_path):
+                os.remove(full_path)
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
+                                     f"Error eliminando el archivo {file_path}: {e}")

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/infra/connectors/s3_connector.py

@@ -32,4 +32,30 @@ class S3Connector(FileConnector):
 
     def get_file_content(self, file_path: str) -> bytes:
         response = self.s3.get_object(Bucket=self.bucket, Key=file_path)
-        return response['Body'].read()
+        return response['Body'].read()
+
+    def delete_file(self, file_path: str) -> None:
+        self.s3.delete_object(Bucket=self.bucket, Key=file_path)
+
+    def upload_file(self, file_path: str, content: bytes, content_type: str = None) -> None:
+        # If the path doesn't start with the prefix, add it (optional, depends on your logic)'
+        # Assuming file_path is either a full path or relative to the root of the bucket for flexibility
+        full_path = file_path
+
+        extra_args = {}
+        if content_type:
+            extra_args['ContentType'] = content_type
+
+        self.s3.put_object(
+            Bucket=self.bucket,
+            Key=full_path,
+            Body=content,
+            **extra_args
+        )
+
+    def generate_presigned_url(self, file_path: str, expiration: int = 3600) -> str:
+        return self.s3.generate_presigned_url(
+            'get_object',
+            Params={'Bucket': self.bucket, 'Key': file_path},
+            ExpiresIn=expiration
+        )

{iatoolkit-1.10.0 → iatoolkit-1.21.0}/src/iatoolkit/infra/llm_providers/deepseek_adapter.py

@@ -40,6 +40,13 @@ class DeepseekAdapter:
         tools = kwargs.get("tools") or []
         tool_choice = kwargs.get("tool_choice", "auto")
         context_history = kwargs.get("context_history") or []
+        images = kwargs.get("images") or []
+
+        if images:
+            logging.warning(
+                f"[DeepseekAdapter] Images provided but DeepSeek models are not multimodal. "
+                f"Ignoring {len(images)} images."
+            )
 
         try:
             # 1) Build messages from history (if any)
@@ -232,6 +239,7 @@ class DeepseekAdapter:
 
         # If the model produced tool calls, fills this list
         tool_calls_out: List[ToolCall] = []
+        content_parts: List[Dict] = []  # Initialize content_parts
 
         tool_calls = getattr(message, "tool_calls", None) or []
         if not tool_calls:
@@ -239,6 +247,13 @@ class DeepseekAdapter:
             output_text = getattr(message, "content", "") or ""
             status = "completed"
 
+            # Fill content_parts for text response
+            if output_text:
+                content_parts.append({
+                    "type": "text",
+                    "text": output_text
+                })
+
         else:
             logging.debug(f"[DeepSeek] RAW tool_calls: {tool_calls}")
 
@@ -274,5 +289,6 @@ class DeepseekAdapter:
             output_text=output_text,
             output=tool_calls_out,
             usage=usage,
-            reasoning_content=reasoning_content
+            reasoning_content=reasoning_content,
+            content_parts=content_parts  # Pass content_parts
         )