evolutia 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

evolutia/rag/rag_indexer.py

@@ -45,41 +45,54 @@ class RAGIndexer:
             base_path: Ruta base del proyecto
             chroma_client: Cliente ChromaDB compartido (opcional)
         """
-        self.config = config
-        self.base_path = Path(base_path)
-        self.vector_store = None
-        self.embedding_model = None
-        self.embedding_provider = config.get('embeddings', {}).get('provider', 'openai')
-        self.chroma_client = chroma_client
-        self._setup_embeddings()
-        self._setup_vector_store()
-    
-    def _setup_embeddings(self):
-        """Configura el modelo de embeddings."""
-        embeddings_config = self.config.get('embeddings', {})
-        provider = embeddings_config.get('provider', 'openai')
-        model_name = embeddings_config.get('model', 'text-embedding-3-small')
-        
-        if provider == 'openai':
-            if not OPENAI_AVAILABLE:
-                raise ImportError("openai no está instalado. Instala con: pip install openai")
-            
-            api_key = os.getenv("OPENAI_API_KEY")
-            if not api_key:
-                raise ValueError("OPENAI_API_KEY no encontrada en variables de entorno")
-            
-            self.embedding_client = OpenAI(api_key=api_key)
-            self.embedding_model_name = model_name
-            logger.info(f"Usando embeddings de OpenAI: {model_name}")
-        
-        elif provider == 'sentence-transformers':
-            if not SENTENCE_TRANSFORMERS_AVAILABLE:
-                raise ImportError("sentence-transformers no está instalado. Instala con: pip install sentence-transformers")
-            
-            self.embedding_model = SentenceTransformer(model_name)
-            logger.info(f"Usando embeddings locales: {model_name}")
-        else:
-            raise ValueError(f"Proveedor de embeddings no soportado: {provider}")
+        self.config = config
+        self.base_path = Path(base_path)
+        self.vector_store = None
+        self.embedding_model = None
+        self.embedding_client = None
+        self.embedding_model_name = None
+        self.embedding_provider = config.get('embeddings', {}).get('provider', 'openai')
+        self.chroma_client = chroma_client
+        self._embeddings_initialized = False
+        self._setup_vector_store()
+
+    def _ensure_embeddings_initialized(self):
+        """
+        Inicializa el modelo de embeddings de forma lazy (solo cuando se necesita).
+        """
+        if self._embeddings_initialized:
+            return
+
+        embeddings_config = self.config.get('embeddings', {})
+        provider = embeddings_config.get('provider', 'openai')
+        model_name = embeddings_config.get('model', 'text-embedding-3-small')
+
+        if provider == 'openai':
+            if not OPENAI_AVAILABLE:
+                raise ImportError("openai no está instalado. Instala con: pip install openai")
+
+            api_key = os.getenv("OPENAI_API_KEY")
+            if not api_key:
+                raise ValueError("OPENAI_API_KEY no encontrada en variables de entorno")
+
+            self.embedding_client = OpenAI(api_key=api_key)
+            self.embedding_model_name = model_name
+            logger.info(f"[RAGIndexer] Inicializados embeddings de OpenAI: {model_name}")
+
+        elif provider == 'sentence-transformers':
+            if not SENTENCE_TRANSFORMERS_AVAILABLE:
+                raise ImportError("sentence-transformers no está instalado. Instala con: pip install sentence-transformers")
+
+            self.embedding_model = SentenceTransformer(model_name)
+            logger.info(f"[RAGIndexer] Inicializados embeddings locales: {model_name}")
+        else:
+            raise ValueError(f"Proveedor de embeddings no soportado: {provider}")
+
+        self._embeddings_initialized = True
+
+    def _setup_embeddings(self):
+        """Configura el modelo de embeddings (mantenido para compatibilidad)."""
+        self._ensure_embeddings_initialized()
     
     def _setup_vector_store(self):
         """Configura el vector store."""
@@ -114,62 +127,66 @@ class RAGIndexer:
             )
             logger.info(f"Nueva colección creada: {collection_name}")
     
-    def _generate_embedding(self, text: str) -> List[float]:
-        """
-        Genera embedding para un texto.
-        
-        Args:
-            text: Texto a convertir en embedding
-            
-        Returns:
-            Lista de floats representando el embedding
-        """
-        if self.embedding_provider == 'openai':
-            response = self.embedding_client.embeddings.create(
-                model=self.embedding_model_name,
-                input=text
-            )
-            return response.data[0].embedding
-        
-        elif self.embedding_provider == 'sentence-transformers':
-            return self.embedding_model.encode(text, show_progress_bar=False).tolist()
+    def _generate_embedding(self, text: str) -> List[float]:
+        """
+        Genera embedding para un texto.
+
+        Args:
+            text: Texto a convertir en embedding
+
+        Returns:
+            Lista de floats representando el embedding
+        """
+        self._ensure_embeddings_initialized()
+
+        if self.embedding_provider == 'openai':
+            response = self.embedding_client.embeddings.create(
+                model=self.embedding_model_name,
+                input=text
+            )
+            return response.data[0].embedding
+
+        elif self.embedding_provider == 'sentence-transformers':
+            return self.embedding_model.encode(text, show_progress_bar=False).tolist()
     
-    def _generate_embeddings_batch(self, texts: List[str]) -> List[List[float]]:
-        """
-        Genera embeddings para múltiples textos en batch.
-        
-        Args:
-            texts: Lista de textos
-            
-        Returns:
-            Lista de embeddings
-        """
-        if self.embedding_provider == 'openai':
-            batch_size = self.config.get('embeddings', {}).get('batch_size', 100)
-            embeddings = []
-            
-            # Filtrar textos vacíos para evitar error 400 de OpenAI
-            valid_texts = [t for t in texts if t and t.strip()]
-            if not valid_texts:
-                return []
-                
-            for i in range(0, len(valid_texts), batch_size):
-                batch = valid_texts[i:i + batch_size]
-                try:
-                    response = self.embedding_client.embeddings.create(
-                        model=self.embedding_model_name,
-                        input=batch
-                    )
-                    embeddings.extend([item.embedding for item in response.data])
-                except Exception as e:
-                    logger.error(f"Error en OpenAI embeddings: {e}")
-                    logger.error(f"Batch problemático: {batch}")
-                    raise
-            
-            return embeddings
-        
-        elif self.embedding_provider == 'sentence-transformers':
-            return self.embedding_model.encode(texts, show_progress_bar=True, batch_size=32).tolist()
+    def _generate_embeddings_batch(self, texts: List[str]) -> List[List[float]]:
+        """
+        Genera embeddings para múltiples textos en batch.
+
+        Args:
+            texts: Lista de textos
+
+        Returns:
+            Lista de embeddings
+        """
+        self._ensure_embeddings_initialized()
+
+        if self.embedding_provider == 'openai':
+            batch_size = self.config.get('embeddings', {}).get('batch_size', 100)
+            embeddings = []
+
+            # Filtrar textos vacíos para evitar error 400 de OpenAI
+            valid_texts = [t for t in texts if t and t.strip()]
+            if not valid_texts:
+                return []
+
+            for i in range(0, len(valid_texts), batch_size):
+                batch = valid_texts[i:i + batch_size]
+                try:
+                    response = self.embedding_client.embeddings.create(
+                        model=self.embedding_model_name,
+                        input=batch
+                    )
+                    embeddings.extend([item.embedding for item in response.data])
+                except Exception as e:
+                    logger.error(f"Error en OpenAI embeddings: {e}")
+                    logger.error(f"Batch problemático: {batch}")
+                    raise
+
+            return embeddings
+
+        elif self.embedding_provider == 'sentence-transformers':
+            return self.embedding_model.encode(texts, show_progress_bar=True, batch_size=32).tolist()
     
     def _chunk_text(self, text: str, chunk_size: int = 1000, overlap: int = 100) -> List[str]:
         """

evolutia/rag/rag_retriever.py

@@ -44,35 +44,48 @@ class RAGRetriever:
             base_path: Ruta base del proyecto
             chroma_client: Cliente ChromaDB compartido (opcional)
         """
-        self.config = config
-        self.base_path = Path(base_path)
-        self.embedding_provider = config.get('embeddings', {}).get('provider', 'openai')
-        self.chroma_client = chroma_client
-        self._setup_embeddings()
-        self._setup_vector_store()
-    
-    def _setup_embeddings(self):
-        """Configura el modelo de embeddings (debe coincidir con el indexer)."""
-        embeddings_config = self.config.get('embeddings', {})
-        provider = embeddings_config.get('provider', 'openai')
-        model_name = embeddings_config.get('model', 'text-embedding-3-small')
-        
-        if provider == 'openai':
-            if not OPENAI_AVAILABLE:
-                raise ImportError("openai no está instalado")
-            
-            api_key = os.getenv("OPENAI_API_KEY")
-            if not api_key:
-                raise ValueError("OPENAI_API_KEY no encontrada")
-            
-            self.embedding_client = OpenAI(api_key=api_key)
-            self.embedding_model_name = model_name
-        
-        elif provider == 'sentence-transformers':
-            if not SENTENCE_TRANSFORMERS_AVAILABLE:
-                raise ImportError("sentence-transformers no está instalado")
-            
-            self.embedding_model = SentenceTransformer(model_name)
+        self.config = config
+        self.base_path = Path(base_path)
+        self.embedding_provider = config.get('embeddings', {}).get('provider', 'openai')
+        self.chroma_client = chroma_client
+        self._embeddings_initialized = False
+        self._setup_vector_store()
+
+    def _ensure_embeddings_initialized(self):
+        """
+        Inicializa el modelo de embeddings de forma lazy (solo cuando se necesita).
+        """
+        if self._embeddings_initialized:
+            return
+
+        embeddings_config = self.config.get('embeddings', {})
+        provider = embeddings_config.get('provider', 'openai')
+        model_name = embeddings_config.get('model', 'text-embedding-3-small')
+
+        if provider == 'openai':
+            if not OPENAI_AVAILABLE:
+                raise ImportError("openai no está instalado")
+
+            api_key = os.getenv("OPENAI_API_KEY")
+            if not api_key:
+                raise ValueError("OPENAI_API_KEY no encontrada")
+
+            self.embedding_client = OpenAI(api_key=api_key)
+            self.embedding_model_name = model_name
+            logger.info(f"[RAGRetriever] Inicializados embeddings de OpenAI: {model_name}")
+
+        elif provider == 'sentence-transformers':
+            if not SENTENCE_TRANSFORMERS_AVAILABLE:
+                raise ImportError("sentence-transformers no está instalado")
+
+            self.embedding_model = SentenceTransformer(model_name)
+            logger.info(f"[RAGRetriever] Inicializados embeddings locales: {model_name}")
+
+        self._embeddings_initialized = True
+
+    def _setup_embeddings(self):
+        """Configura el modelo de embeddings (mantenido para compatibilidad)."""
+        self._ensure_embeddings_initialized()
     
     def _setup_vector_store(self):
         """Configura la conexión al vector store."""
@@ -117,58 +130,64 @@ class RAGRetriever:
         elif self.embedding_provider == 'sentence-transformers':
             return self.embedding_model.encode(query, show_progress_bar=False).tolist()
     
-    def retrieve_similar_exercises(self, exercise_content: str, top_k: int = 5,
-                                   exclude_label: Optional[str] = None,
-                                   min_complexity: Optional[float] = None,
-                                   max_complexity: Optional[float] = None) -> List[Dict]:
-        """
-        Recupera ejercicios similares al contenido dado.
-        
-        Args:
-            exercise_content: Contenido del ejercicio de referencia
-            top_k: Número de resultados a recuperar
-            exclude_label: Label del ejercicio a excluir (el original)
-            min_complexity: Complejidad mínima
-            max_complexity: Complejidad máxima
-            
-        Returns:
-            Lista de ejercicios similares con sus metadatos
-        """
-        retrieval_config = self.config.get('retrieval', {})
-        top_k = retrieval_config.get('top_k', top_k)
-        similarity_threshold = retrieval_config.get('similarity_threshold', 0.7)
-        
-        # Generar embedding del query
-        query_embedding = self._generate_query_embedding(exercise_content)
-        
-        # Construir filtros de metadatos usando sintaxis correcta de ChromaDB
-        conditions = [{'type': 'exercise'}]
-        
-        if exclude_label:
-            conditions.append({'label': {'$ne': exclude_label}})
-        
-        if min_complexity is not None and max_complexity is not None:
-            conditions.append({'complexity': {'$gte': float(min_complexity)}})
-            conditions.append({'complexity': {'$lte': float(max_complexity)}})
-        elif min_complexity is not None:
-            conditions.append({'complexity': {'$gte': float(min_complexity)}})
-        elif max_complexity is not None:
-            conditions.append({'complexity': {'$lte': float(max_complexity)}})
-        
-        # Si hay múltiples condiciones, usar $and
-        if len(conditions) > 1:
-            where = {'$and': conditions}
-        elif len(conditions) == 1:
-            where = conditions[0]
-        else:
-            where = None
-        
-        # Buscar en el vector store
-        results = self.collection.query(
-            query_embeddings=[query_embedding],
-            n_results=top_k * 2,  # Buscar más para filtrar después
-            where=where
-        )
+    def retrieve_similar_exercises(self, exercise_content: str, top_k: int = 5,
+                                   exclude_label: Optional[str] = None,
+                                   min_complexity: Optional[float] = None,
+                                   max_complexity: Optional[float] = None) -> List[Dict]:
+        """
+        Recupera ejercicios similares al contenido dado.
+
+        Args:
+            exercise_content: Contenido del ejercicio de referencia
+            top_k: Número de resultados a recuperar
+            exclude_label: Label del ejercicio a excluir (el original)
+            min_complexity: Complejidad mínima
+            max_complexity: Complejidad máxima
+
+        Returns:
+            Lista de ejercicios similares con sus metadatos
+        """
+        self._ensure_embeddings_initialized()
+
+        retrieval_config = self.config.get('retrieval', {})
+        top_k = retrieval_config.get('top_k', top_k)
+        similarity_threshold = retrieval_config.get('similarity_threshold', 0.7)
+        max_results_limit = retrieval_config.get('max_results_limit', 100)  # Límite absoluto
+
+        # Generar embedding del query
+        query_embedding = self._generate_query_embedding(exercise_content)
+
+        # Construir filtros de metadatos usando sintaxis correcta de ChromaDB
+        conditions = [{'type': 'exercise'}]
+
+        if exclude_label:
+            conditions.append({'label': {'$ne': exclude_label}})
+
+        if min_complexity is not None and max_complexity is not None:
+            conditions.append({'complexity': {'$gte': float(min_complexity)}})
+            conditions.append({'complexity': {'$lte': float(max_complexity)}})
+        elif min_complexity is not None:
+            conditions.append({'complexity': {'$gte': float(min_complexity)}})
+        elif max_complexity is not None:
+            conditions.append({'complexity': {'$lte': float(max_complexity)}})
+
+        # Si hay múltiples condiciones, usar $and
+        if len(conditions) > 1:
+            where = {'$and': conditions}
+        elif len(conditions) == 1:
+            where = conditions[0]
+        else:
+            where = None
+
+        # Calcular número de resultados a buscar con límite absoluto
+        n_results = min(top_k * 2, max_results_limit)
+
+        # Buscar en el vector store
+        results = self.collection.query(
+            query_embeddings=[query_embedding],
+            n_results=n_results,  # Buscar más para filtrar después, pero con límite
+            where=where
+        )
         
         # Procesar resultados
         similar_exercises = []
@@ -323,28 +342,36 @@ class RAGRetriever:
         logger.info(f"Recuperados {len(exercises)} ejercicios por complejidad")
         return exercises
     
-    def hybrid_search(self, query: str, metadata_filters: Dict = None,
-                     top_k: int = 5) -> List[Dict]:
-        """
-        Búsqueda híbrida: semántica + filtros de metadatos.
-        
-        Args:
-            query: Consulta de texto
-            metadata_filters: Filtros de metadatos (ej: {'type': 'exercise'})
-            top_k: Número de resultados
-            
-        Returns:
-            Lista de resultados
-        """
-        query_embedding = self._generate_query_embedding(query)
-        
-        where = metadata_filters or {}
-        
-        results = self.collection.query(
-            query_embeddings=[query_embedding],
-            n_results=top_k,
-            where=where if where else None
-        )
+    def hybrid_search(self, query: str, metadata_filters: Dict = None,
+                     top_k: int = 5) -> List[Dict]:
+        """
+        Búsqueda híbrida: semántica + filtros de metadatos.
+
+        Args:
+            query: Consulta de texto
+            metadata_filters: Filtros de metadatos (ej: {'type': 'exercise'})
+            top_k: Número de resultados
+
+        Returns:
+            Lista de resultados
+        """
+        self._ensure_embeddings_initialized()
+
+        retrieval_config = self.config.get('retrieval', {})
+        max_results_limit = retrieval_config.get('max_results_limit', 100)  # Límite absoluto
+
+        query_embedding = self._generate_query_embedding(query)
+
+        where = metadata_filters or {}
+
+        # Calcular número de resultados con límite absoluto
+        n_results = min(top_k, max_results_limit)
+
+        results = self.collection.query(
+            query_embeddings=[query_embedding],
+            n_results=n_results,
+            where=where if where else None
+        )
         
         hybrid_results = []