bertrend 0.3.22__tar.gz → 0.3.23__tar.gz

{bertrend-0.3.22 → bertrend-0.3.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.3.22
+Version: 0.3.23
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -42,7 +42,7 @@ Requires-Dist: langdetect (>=1.0.9)
 Requires-Dist: loguru (>=0.7.3)
 Requires-Dist: lxml-html-clean (>=0.2.2)
 Requires-Dist: markdown (>=3.7)
-Requires-Dist: newscatcherapi (>=0.7.3) ; extra == "apps"
+Requires-Dist: newscatcher-sdk (>=1.4.0) ; extra == "apps"
 Requires-Dist: newspaper4k (>=0.9.3.1) ; extra == "apps"
 Requires-Dist: nltk (>=3.9.1)
 Requires-Dist: numpy (<2)

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/llm_utils/openai_client.py

@@ -8,17 +8,12 @@ import re
 from enum import Enum
 from typing import Type
 
-from dotenv import load_dotenv
 from openai import OpenAI, AzureOpenAI, Timeout, Stream
 from loguru import logger
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from pydantic import BaseModel
 
-# Load environment variables at module import
-if load_dotenv(override=True):
-    logger.info("Loaded .env file")
-else:
-    logger.warning("Failed to load .env file")
+# Note: .env is loaded in bertrend/__init__.py which is imported before this module
 
 MAX_ATTEMPTS = 3
 TIMEOUT = 60.0

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/services/embedding_server/start.py

@@ -4,16 +4,19 @@
 #  This file is part of BERTrend.
 
 import os
-import uvicorn
 
+# Load the configuration BEFORE any other imports that might use CUDA
 from bertrend.services.embedding_server.config.settings import get_config
 
-# Load the configuration
 CONFIG = get_config()
 
-# Set the CUDA_VISIBLE_DEVICES environment variable
+# Set the CUDA_VISIBLE_DEVICES environment variable BEFORE importing uvicorn
+# This is critical because uvicorn will import the app module, which imports torch
+# We override the value with the content of the config
 os.environ["CUDA_VISIBLE_DEVICES"] = CONFIG.cuda_visible_devices
 
+import uvicorn
+
 # Start the FastAPI application
 if __name__ == "__main__":
     uvicorn.run(

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/services/scheduling/docker-compose.yml

@@ -4,7 +4,6 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: bertrend-scheduler
-    user: "${HOST_UID:-1000}:${HOST_GID:-1000}"
     ports:
       - "8882:8000"
     volumes:

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/services/scheduling/job_utils/job_functions.py

@@ -5,7 +5,6 @@
 import time
 
 import requests
-from fastapi import HTTPException
 from loguru import logger
 
 
@@ -24,7 +23,7 @@ def http_request(
     method: str = "GET",
     headers: dict = None,
     json_data: dict = None,
-    timeout: int = 30,
+    timeout: int = 600,  # 10 min
 ):
     """Execute an HTTP request (curl-like functionality)"""
     logger.info(f"Executing HTTP {method} request to {url}")
@@ -42,9 +41,7 @@ def http_request(
         return f"Request to {url} completed with status {response.status_code}"
     except Exception as e:
         logger.error(f"HTTP request failed: {str(e)}")
-        raise HTTPException(
-            status_code=500, detail=f"Error executing request: {str(e)}"
-        )
+        raise RuntimeError(f"Error executing request: {str(e)}")
 
 
 # Job function registry

bertrend-0.3.23/bertrend/tests/apps/test_lazy_scheduler.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+"""
+Test script to verify lazy initialization of SCHEDULER_UTILS.
+This tests that importing bertrend_apps doesn't immediately create scheduler instances.
+"""
+import os
+import sys
+import multiprocessing
+
+
+def helper_test_lazy_init_in_worker():
+    """Test that SCHEDULER_UTILS is lazily initialized in worker processes."""
+    try:
+        pid = os.getpid()
+        print(f"[Worker {pid}] Importing bertrend_apps...")
+
+        # Import should not trigger initialization
+        from bertrend_apps import SCHEDULER_UTILS
+
+        print(f"[Worker {pid}] Import complete (should not have initialized yet)")
+
+        # Now actually use it - this should trigger initialization
+        print(f"[Worker {pid}] Accessing SCHEDULER_UTILS...")
+        scheduler_type = type(SCHEDULER_UTILS).__name__
+        print(f"[Worker {pid}] Successfully accessed SCHEDULER_UTILS: {scheduler_type}")
+
+        # Try to call a method to ensure proxy works
+        if hasattr(SCHEDULER_UTILS, "add_job_to_crontab"):
+            print(f"[Worker {pid}] Proxy working correctly - method accessible")
+
+        return True
+    except Exception as e:
+        print(f"[Worker {pid}] Failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+
+def test_lazy_init_in_worker():
+    """Main test function."""
+    print(f"[Main {os.getpid()}] Starting lazy initialization test...")
+
+    # Test with multiple processes (simulating uvicorn workers)
+    num_workers = 2
+    processes = []
+
+    for i in range(num_workers):
+        p = multiprocessing.Process(target=helper_test_lazy_init_in_worker)
+        p.start()
+        processes.append(p)
+
+    # Wait for all processes to complete
+    success = True
+    for p in processes:
+        p.join()
+        if p.exitcode != 0:
+            success = False
+            print(f"[Main] Worker process {p.pid} exited with code {p.exitcode}")
+
+    if success:
+        print("[Main] ✓ All workers completed successfully with lazy initialization!")
+    else:
+        print("[Main] ✗ Some workers failed!")
+
+    assert success == True

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/tests/apps/test_models.py

@@ -14,8 +14,6 @@ from bertrend_apps.services.models.bertrend_app_models import (
 )
 from bertrend_apps.services.models.newsletters_models import (
     NewsletterRequest,
-    NewsletterResponse,
-    ScheduleNewsletterRequest,
 )
 from bertrend_apps.services.models.data_provider_models import (
     ScrapeRequest,
@@ -83,32 +81,15 @@ class TestNewslettersModels:
         with pytest.raises(ValidationError):
             NewsletterRequest(newsletter_toml_path=Path("test.toml"))
 
-    def test_newsletter_response_valid(self, tmp_path):
-        output_path = tmp_path / "output.html"
-        resp = NewsletterResponse(output_path=output_path, status="success")
-        assert resp.output_path == output_path
-        assert resp.status == "success"
-
     def test_schedule_newsletter_request_valid(self, tmp_path):
         newsletter_path = tmp_path / "newsletter.toml"
         feed_path = tmp_path / "feed.toml"
-        req = ScheduleNewsletterRequest(
-            newsletter_toml_cfg_path=newsletter_path,
-            data_feed_toml_cfg_path=feed_path,
-        )
-        assert req.newsletter_toml_cfg_path == newsletter_path
-        assert req.data_feed_toml_cfg_path == feed_path
-        assert req.cuda_devices is None  # default value
-
-    def test_schedule_newsletter_request_with_cuda(self, tmp_path):
-        newsletter_path = tmp_path / "newsletter.toml"
-        feed_path = tmp_path / "feed.toml"
-        req = ScheduleNewsletterRequest(
-            newsletter_toml_cfg_path=newsletter_path,
-            data_feed_toml_cfg_path=feed_path,
-            cuda_devices="0,1",
+        req = NewsletterRequest(
+            newsletter_toml_path=newsletter_path,
+            data_feed_toml_path=feed_path,
         )
-        assert req.cuda_devices == "0,1"
+        assert req.newsletter_toml_path == newsletter_path
+        assert req.data_feed_toml_path == feed_path
 
 
 class TestDataProviderModels:

bertrend-0.3.23/bertrend/tests/apps/test_multiworker.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+"""
+Test script to verify the multi-worker fix.
+This script simulates what happens when uvicorn forks worker processes.
+"""
+import os
+import sys
+import multiprocessing
+
+
+def helper_test_import_in_worker():
+    """Test that importing bertrend_apps works in a forked worker process."""
+    try:
+        # Import bertrend_apps which triggers SCHEDULER_UTILS initialization
+        from bertrend_apps import SCHEDULER_UTILS
+
+        print(
+            f"[Worker {os.getpid()}] Successfully imported SCHEDULER_UTILS: {type(SCHEDULER_UTILS).__name__}"
+        )
+        return True
+    except Exception as e:
+        print(f"[Worker {os.getpid()}] Failed to import: {e}")
+        return False
+
+
+def test_multiworker():
+    """Main test function."""
+    print(f"[Main {os.getpid()}] Starting multi-worker import test...")
+
+    # Test with multiple processes (simulating uvicorn workers)
+    num_workers = 2
+    processes = []
+
+    for i in range(num_workers):
+        p = multiprocessing.Process(target=helper_test_import_in_worker)
+        p.start()
+        processes.append(p)
+
+    # Wait for all processes to complete
+    success = True
+    for p in processes:
+        p.join()
+        if p.exitcode != 0:
+            success = False
+            print(f"[Main] Worker process {p.pid} exited with code {p.exitcode}")
+
+    if success:
+        print("[Main] ✓ All workers completed successfully!")
+    else:
+        print("[Main] ✗ Some workers failed!")
+
+    assert success

bertrend-0.3.23/bertrend/tests/services/scheduling/test_job_exception_fix.py

@@ -0,0 +1,74 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+"""
+Pytest tests to verify that job_functions.http_request properly raises
+RuntimeError instead of HTTPException, which fixes the pickling issue
+in multiprocessing contexts.
+"""
+
+import pickle
+import pytest
+from concurrent.futures import ProcessPoolExecutor
+
+from bertrend.services.scheduling.job_utils.job_functions import http_request
+
+
+def job_wrapper_for_pool():
+    """Module-level wrapper to call http_request in a subprocess"""
+    try:
+        http_request("http://invalid-host-that-does-not-exist-12345.com", timeout=1)
+    except RuntimeError as e:
+        return f"RuntimeError: {str(e)[:50]}"
+    except Exception as e:
+        return f"OtherError: {type(e).__name__}: {str(e)[:50]}"
+
+
+def test_exception_is_picklable():
+    """Test that the exception raised by http_request can be pickled"""
+    with pytest.raises(RuntimeError) as exc_info:
+        # Try to trigger an exception by connecting to an invalid URL
+        http_request("http://invalid-host-that-does-not-exist-12345.com", timeout=1)
+
+    # Verify the exception was caught
+    exception = exc_info.value
+    assert isinstance(exception, RuntimeError)
+
+    # Try to pickle the exception
+    pickled = pickle.dumps(exception)
+    unpickled = pickle.loads(pickled)
+
+    # Verify unpickling worked correctly
+    assert isinstance(unpickled, RuntimeError)
+    assert type(exception).__name__ == type(unpickled).__name__
+
+
+def test_exception_in_process_pool():
+    """Test that http_request exceptions work properly in ProcessPoolExecutor"""
+    with ProcessPoolExecutor(max_workers=1) as executor:
+        future = executor.submit(job_wrapper_for_pool)
+        result = future.result(timeout=10)
+
+        # Verify that the RuntimeError was properly handled across process boundary
+        assert "RuntimeError" in result
+        assert "Error executing request" in result
+
+
+def test_timeout_scenario():
+    """Test with a timeout scenario (simulates the original error)"""
+    with pytest.raises(RuntimeError) as exc_info:
+        # Use a valid host but very short timeout to trigger timeout
+        http_request("https://www.google.com", timeout=0.001)
+
+    error_msg = str(exc_info.value)
+
+    # Verify it's a timeout-related or network-related error
+    # (GitHub runners may have no network access, causing "network is unreachable" instead of timeout)
+    assert (
+        "timeout" in error_msg.lower()
+        or "timed out" in error_msg.lower()
+        or "network is unreachable" in error_msg.lower()
+    )
+    assert "Error executing request" in error_msg

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/tests/utils/test_apscheduler_utils.py

@@ -107,23 +107,17 @@ class TestHelperFunctions:
 class TestAPSchedulerUtils:
     """Tests for APSchedulerUtils class."""
 
-    @patch("bertrend_apps.common.apscheduler_utils._request")
-    def test_init_success(self, mock_request):
+    def test_init_success(self):
         """Test successful initialization."""
-        mock_response = MagicMock()
-        mock_response.status_code = 200
-        mock_request.return_value = mock_response
-
+        # Health check removed from __init__ to avoid blocking worker startup
         scheduler = APSchedulerUtils()
 
-        mock_request.assert_called_once_with("GET", "/health")
+        assert scheduler is not None
 
-    @patch("bertrend_apps.common.apscheduler_utils._request")
-    def test_init_service_unavailable(self, mock_request):
+    def test_init_service_unavailable(self):
         """Test initialization when service is unavailable."""
-        mock_request.side_effect = Exception("Connection error")
-
-        # Should not raise, just log error
+        # Health check removed from __init__ to avoid blocking worker startup
+        # Initialization should always succeed regardless of service availability
         scheduler = APSchedulerUtils()
 
         assert scheduler is not None
@@ -233,15 +227,11 @@ class TestAPSchedulerUtils:
     @patch("bertrend_apps.common.apscheduler_utils._request")
     def test_add_job_to_crontab_success(self, mock_request):
         """Test successfully adding a job."""
-        # Mock health check
-        mock_health_response = MagicMock()
-        mock_health_response.status_code = 200
-
         # Mock job creation
         mock_job_response = MagicMock()
         mock_job_response.status_code = 201
 
-        mock_request.side_effect = [mock_health_response, mock_job_response]
+        mock_request.return_value = mock_job_response
 
         scheduler = APSchedulerUtils()
         result = scheduler.add_job_to_crontab(
@@ -251,7 +241,7 @@ class TestAPSchedulerUtils:
         )
 
         assert result is True
-        assert mock_request.call_count == 2
+        assert mock_request.call_count == 1
 
     @patch("bertrend_apps.common.apscheduler_utils._request")
     def test_add_job_to_crontab_with_kwargs(self, mock_request):
@@ -280,15 +270,12 @@ class TestAPSchedulerUtils:
     @patch("bertrend_apps.common.apscheduler_utils._request")
     def test_add_job_to_crontab_failure(self, mock_request):
         """Test adding a job when request fails."""
-        mock_health_response = MagicMock()
-        mock_health_response.status_code = 200
-
         mock_job_response = MagicMock()
         mock_job_response.status_code = 400
         mock_job_response.text = "Bad request"
         mock_job_response.json.return_value = {"detail": "Invalid data"}
 
-        mock_request.side_effect = [mock_health_response, mock_job_response]
+        mock_request.return_value = mock_job_response
 
         scheduler = APSchedulerUtils()
         result = scheduler.add_job_to_crontab(
@@ -310,20 +297,17 @@ class TestAPSchedulerUtils:
         }
         mock_load_config.return_value = mock_config
 
-        mock_health_response = MagicMock()
-        mock_health_response.status_code = 200
-
         mock_job_response = MagicMock()
         mock_job_response.status_code = 201
 
-        mock_request.side_effect = [mock_health_response, mock_job_response]
+        mock_request.return_value = mock_job_response
 
         scheduler = APSchedulerUtils()
         feed_cfg = Path("/path/to/feed.toml")
         scheduler.schedule_scrapping(feed_cfg, user="testuser")
 
         # Verify job creation was called
-        assert mock_request.call_count == 2
+        assert mock_request.call_count == 1
 
     @patch("bertrend_apps.common.apscheduler_utils.load_toml_config")
     @patch("bertrend_apps.common.apscheduler_utils._request")
@@ -337,20 +321,17 @@ class TestAPSchedulerUtils:
         }
         mock_load_config.return_value = mock_config
 
-        mock_health_response = MagicMock()
-        mock_health_response.status_code = 200
-
         mock_job_response = MagicMock()
         mock_job_response.status_code = 200
 
-        mock_request.side_effect = [mock_health_response, mock_job_response]
+        mock_request.return_value = mock_job_response
 
         scheduler = APSchedulerUtils()
         newsletter_cfg = Path("/path/to/newsletter.toml")
         feed_cfg = Path("/path/to/feed.toml")
-        scheduler.schedule_newsletter(newsletter_cfg, feed_cfg, cuda_devices="1")
+        scheduler.schedule_newsletter(newsletter_cfg, feed_cfg)
 
-        assert mock_request.call_count == 2
+        assert mock_request.call_count == 1
 
     @patch("bertrend_apps.common.apscheduler_utils._request")
     def test_schedule_training_for_user(self, mock_request):
@@ -400,10 +381,6 @@ class TestAPSchedulerUtils:
     @patch("bertrend_apps.common.apscheduler_utils._request")
     def test_schedule_report_generation_auto_send_disabled(self, mock_request):
         """Test that report generation is not scheduled when auto_send is disabled."""
-        mock_health_response = MagicMock()
-        mock_health_response.status_code = 200
-        mock_request.return_value = mock_health_response
-
         scheduler = APSchedulerUtils()
         report_config = {
             "auto_send": False,
@@ -417,8 +394,8 @@ class TestAPSchedulerUtils:
         )
 
         assert result is False
-        # Only health check should be called
-        assert mock_request.call_count == 1
+        # No job creation should happen when auto_send is disabled
+        assert mock_request.call_count == 0
 
     @patch("bertrend_apps.common.apscheduler_utils._request")
     def test_schedule_report_generation_no_recipients(self, mock_request):

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/tests/utils/test_crontab_utils.py

@@ -190,7 +190,6 @@ class TestCrontabSchedulerUtils:
         "bertrend_apps.common.crontab_utils.CrontabSchedulerUtils.add_job_to_crontab"
     )
     @patch("bertrend_apps.common.crontab_utils.BERTREND_LOG_PATH", Path("/logs"))
-    @patch("bertrend_apps.common.crontab_utils.BEST_CUDA_DEVICE", "0")
     def test_schedule_newsletter(self, mock_add_job, mock_load_config):
         """Test scheduling a newsletter job."""
         mock_config = {
@@ -204,13 +203,12 @@ class TestCrontabSchedulerUtils:
 
         newsletter_cfg = Path("/path/to/newsletter.toml")
         feed_cfg = Path("/path/to/feed.toml")
-        self.scheduler.schedule_newsletter(newsletter_cfg, feed_cfg, cuda_devices="0")
+        self.scheduler.schedule_newsletter(newsletter_cfg, feed_cfg)
 
         mock_add_job.assert_called_once()
         call_args = mock_add_job.call_args
         assert call_args[0][0] == "0 8 * * *"
         assert "newsletters" in call_args[0][1]
-        assert "CUDA_VISIBLE_DEVICES=0" in call_args[0][2]
 
     @patch(
         "bertrend_apps.common.crontab_utils.CrontabSchedulerUtils.add_job_to_crontab"

{bertrend-0.3.22 → bertrend-0.3.23}/bertrend/utils/data_loading.py

@@ -106,9 +106,15 @@ def load_data(
         language (str): The language of the text data ('French' or 'English').
 
     Returns:
-        pd.DataFrame: The loaded and preprocessed DataFrame.
+        pd.DataFrame: The loaded and preprocessed DataFrame, or None if the file is empty.
     """
     df = _file_to_pd(selected_file)
+
+    # Check if the DataFrame is empty (no rows or no columns)
+    if df is None or df.empty or len(df.columns) == 0:
+        logger.warning(f"Skipping empty file: {selected_file.name}")
+        return None
+
     _check_data(df, selected_file)
     return _clean_data(df, language)
 

bertrend-0.3.23/bertrend_apps/__init__.py

@@ -0,0 +1,43 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from bertrend_apps.common.apscheduler_utils import APSchedulerUtils
+from bertrend_apps.common.crontab_utils import CrontabSchedulerUtils
+
+
+# Lazy initialization to avoid fork issues with multiple workers
+_scheduler_utils = None
+
+
+def _get_scheduler_utils():
+    """Get or create the scheduler utils instance for the current process."""
+    global _scheduler_utils
+    if _scheduler_utils is None:
+        # Note: .env is already loaded in bertrend/__init__.py
+        # Load scheduler type and URL from environment variables
+        scheduler_type = os.getenv("SCHEDULER_SERVICE_TYPE", "crontab").lower()
+
+        if scheduler_type == "apscheduler":
+            _scheduler_utils = APSchedulerUtils()
+        else:  # assume default="crontab":
+            _scheduler_utils = CrontabSchedulerUtils()
+            logger.info("Using CrontabScheduler")
+
+    return _scheduler_utils
+
+
+# Provide SCHEDULER_UTILS as a property-like access for backward compatibility
+class _SchedulerUtilsProxy:
+    """Proxy to provide lazy initialization while maintaining the same API."""
+
+    def __getattr__(self, name):
+        return getattr(_get_scheduler_utils(), name)
+
+
+SCHEDULER_UTILS = _SchedulerUtilsProxy()