content-core 0.2.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

content_core/__init__.py

@@ -7,18 +7,21 @@ from xml.etree import ElementTree as ET
 
 from dicttoxml import dicttoxml  # type: ignore
 from dotenv import load_dotenv
-from loguru import logger
 
 from content_core.common import ProcessSourceInput
 from content_core.content.cleanup import cleanup_content
 from content_core.content.extraction import extract_content
 from content_core.content.summary import summarize
+from content_core.logging import configure_logging, logger
+
+# Exposing functions for direct access when importing content_core as cc
+extract = extract_content
+clean = cleanup_content
 
 load_dotenv()
 
-# Configure loguru logger
-logger.remove()  # Remove default handler
-logger.add(sys.stderr, level="INFO")  # Default to INFO level
+# Configure loguru logger using centralized configuration
+configure_logging(debug=False)
 
 
 def parse_content_format(content: str) -> str:
@@ -94,10 +97,9 @@ async def ccore_main():
 
     args = parser.parse_args()
 
-    # Adjust logging level based on debug flag
+    # Adjust logging level based on debug flag using centralized configuration
+    configure_logging(debug=args.debug)
     if args.debug:
-        logger.remove()
-        logger.add(sys.stderr, level="DEBUG")
         logger.debug("Debug logging enabled")
 
     content = get_content(args, parser)
@@ -136,10 +138,9 @@ async def cclean_main():
 
     args = parser.parse_args()
 
-    # Adjust logging level based on debug flag
+    # Adjust logging level based on debug flag using centralized configuration
+    configure_logging(debug=args.debug)
     if args.debug:
-        logger.remove()
-        logger.add(sys.stderr, level="DEBUG")
         logger.debug("Debug logging enabled")
 
     content = get_content(args, parser)
@@ -176,10 +177,9 @@ async def csum_main():
 
     args = parser.parse_args()
 
-    # Adjust logging level based on debug flag
+    # Adjust logging level based on debug flag using centralized configuration
+    configure_logging(debug=args.debug)
     if args.debug:
-        logger.remove()
-        logger.add(sys.stderr, level="DEBUG")
         logger.debug("Debug logging enabled")
 
     content = get_content(args, parser)

content_core/config.py

@@ -1,37 +1,27 @@
-from esperanto import AIFactory
-from esperanto.providers.stt import SpeechToTextModel
+import os
+import pkgutil
 
-SPEECH_TO_TEXT_MODEL: SpeechToTextModel = AIFactory.create_speech_to_text(
-    "openai", "whisper-1"
-)
+import yaml
+from dotenv import load_dotenv
 
-DEFAULT_MODEL = AIFactory.create_language(
-    "openai",
-    "gpt-4o-mini",
-    config={
-        "temperature": 0.5,
-        "top_p": 1,
-        "max_tokens": 2000,
-    },
-)
+# Load environment variables from .env file
+load_dotenv()
 
-CLEANUP_MODEL = AIFactory.create_language(
-    "openai",
-    "gpt-4o-mini",
-    config={
-        "temperature": 0,
-        "max_tokens": 8000,
-        "output_format": "json",
-        # "stream": True, # TODO: handle streaming
-    },
-)  # Fix deprecation
 
-SUMMARY_MODEL = AIFactory.create_language(
-    "openai",
-    "gpt-4o-mini",
-    config={
-        "temperature": 0,
-        "top_p": 1,
-        "max_tokens": 2000,
-    },
-)
+def load_config():
+    config_path = os.environ.get("CCORE_MODEL_CONFIG_PATH")
+    if config_path and os.path.exists(config_path):
+        try:
+            with open(config_path, "r") as file:
+                return yaml.safe_load(file)
+        except Exception as e:
+            print(f"Erro ao carregar o arquivo de configuração de {config_path}: {e}")
+            print("Usando configurações padrão internas.")
+
+    default_config_data = pkgutil.get_data("content_core", "models_config.yaml")
+    if default_config_data:
+        return yaml.safe_load(default_config_data)
+    return {}
+
+
+CONFIG = load_config()

content_core/content/__init__.py

@@ -0,0 +1,5 @@
+from .cleanup import cleanup_content
+from .extraction import extract_content
+from .summary import summarize
+
+__all__ = ["extract_content", "cleanup_content", "summarize"]

content_core/content/cleanup/core.py

@@ -1,11 +1,11 @@
 from functools import partial
 
-from content_core.config import CLEANUP_MODEL
+from content_core.models import ModelFactory
 from content_core.templated_message import TemplatedMessageInput, templated_message
 
 
 async def cleanup_content(content) -> str:
-    templated_summary_fn = partial(templated_message, model=CLEANUP_MODEL)
+    templated_summary_fn = partial(templated_message, model=ModelFactory.get_model('cleanup_model'))
     input = TemplatedMessageInput(
         system_prompt_template="content/cleanup",
         user_prompt_text=content,

content_core/content/extraction/graph.py

@@ -3,13 +3,13 @@ from typing import Any, Dict, Optional
 
 import magic
 from langgraph.graph import END, START, StateGraph
-from loguru import logger
 
 from content_core.common import (
     ProcessSourceInput,
     ProcessSourceState,
     UnsupportedTypeException,
 )
+from content_core.logging import logger
 from content_core.processors.audio import extract_audio  # type: ignore
 from content_core.processors.office import (
     SUPPORTED_OFFICE_TYPES,

content_core/content/summary/core.py

@@ -1,11 +1,11 @@
 from functools import partial
 
-from content_core.config import SUMMARY_MODEL
+from content_core.models import ModelFactory
 from content_core.templated_message import TemplatedMessageInput, templated_message
 
 
 async def summarize(content: str, context: str) -> str:
-    templated_message_fn = partial(templated_message, model=SUMMARY_MODEL)
+    templated_message_fn = partial(templated_message, model=ModelFactory.get_model('summary_model'))
     response = await templated_message_fn(
         TemplatedMessageInput(
             user_prompt_template="content/summarize",

content_core/logging.py

@@ -0,0 +1,15 @@
+import sys
+from loguru import logger
+
+def configure_logging(debug=False):
+    """
+    Configure the global logger for the application.
+    
+    Args:
+        debug (bool): If True, set logging level to DEBUG; otherwise, set to INFO.
+    """
+    logger.remove()  # Remove any existing handlers
+    logger.add(sys.stderr, level="DEBUG" if debug else "INFO")
+
+# Initial configuration with default level (INFO)
+configure_logging(debug=False)

content_core/models.py

@@ -0,0 +1,24 @@
+from esperanto import AIFactory
+from esperanto.providers.stt import SpeechToTextModel
+from .config import CONFIG
+
+class ModelFactory:
+    _instances = {}
+
+    @staticmethod
+    def get_model(model_alias):
+        if model_alias not in ModelFactory._instances:
+            config = CONFIG.get(model_alias, {})
+            if not config:
+                raise ValueError(f"Configuração para o modelo {model_alias} não encontrada.")
+
+            provider = config.get('provider')
+            model_name = config.get('model_name')
+            model_config = config.get('config', {})
+
+            if model_alias == 'speech_to_text':
+                ModelFactory._instances[model_alias] = AIFactory.create_speech_to_text(provider, model_name)
+            else:
+                ModelFactory._instances[model_alias] = AIFactory.create_language(provider, model_name, config=model_config)
+
+        return ModelFactory._instances[model_alias]

content_core/models_config.yaml

@@ -0,0 +1,27 @@
+speech_to_text:
+  provider: openai
+  model_name: whisper-1
+
+default_model:
+  provider: openai
+  model_name: gpt-4o-mini
+  config:
+    temperature: 0.5
+    top_p: 1
+    max_tokens: 2000
+
+cleanup_model:
+  provider: openai
+  model_name: gpt-4o-mini
+  config:
+    temperature: 0
+    max_tokens: 8000
+    output_format: json
+
+summary_model:
+  provider: openai
+  model_name: gpt-4o-mini
+  config:
+    temperature: 0
+    top_p: 1
+    max_tokens: 2000