bertrend 0.3.20__tar.gz → 0.3.22__tar.gz

{bertrend-0.3.20 → bertrend-0.3.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.3.20
+Version: 0.3.22
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -13,8 +13,10 @@ Maintainer-email: jerome.picault@rte-france.com
 Requires-Python: >=3.12,<4.0
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Provides-Extra: apps
+Requires-Dist: APScheduler (>=3.11.0)
 Requires-Dist: PyJWT (>=2.10.1)
 Requires-Dist: Pyqtree (==1.0.0)
+Requires-Dist: SQLAlchemy (>=2.0.40)
 Requires-Dist: arxiv (>=2.2.0) ; extra == "apps"
 Requires-Dist: bertopic (==0.16.2)
 Requires-Dist: black (>=24.10.0)
@@ -24,6 +26,7 @@ Requires-Dist: dask (==2024.12.0)
 Requires-Dist: datamapplot (==0.3.0)
 Requires-Dist: dateparser (>=1.2.0)
 Requires-Dist: dill (>=0.4.0)
+Requires-Dist: fastapi (>=0.120.0)
 Requires-Dist: feedparser (>=6.0.11) ; extra == "apps"
 Requires-Dist: gensim (>=4.3.3)
 Requires-Dist: google-api-python-client (>=2.148.0) ; extra == "apps"

{bertrend-0.3.20 → bertrend-0.3.22}/bertrend/__init__.py

@@ -3,20 +3,39 @@
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
 import os
+import sys
 from pathlib import Path
 
 from loguru import logger
 
+
+# Global flag to track initialization
+_initialized = False
+
 # Auto-load .env if python-dotenv is available
-try:
-    from dotenv import load_dotenv  # type: ignore
+if not _initialized:
+    try:
+        from dotenv import load_dotenv  # type: ignore
+
+        if load_dotenv(override=True):
+            logger.info("Loaded .env file")
+        else:
+            logger.warning("Failed to load .env file")
+
+    except Exception:
+        pass
 
-    if load_dotenv(override=True):
-        logger.info("Loaded .env file")
-    else:
-        logger.warning("Failed to load .env file")
-except Exception:
-    pass
+    # First, remove the default handler (Loguru always adds one by default)
+    logger.remove()
+
+    # Add back a colorized stdout sink
+    logger.add(
+        sys.stdout,
+        colorize=True,
+        format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+    )
+
+    _initialized = True
 
 from bertrend.utils.config_utils import load_toml_config
 
@@ -43,11 +62,7 @@ LLM_CONFIG = SERVICES_CONFIG["llm_service"]
 BEST_CUDA_DEVICE = r"\`nvidia-smi --query-gpu=index,memory.used --format=csv,nounits | tail -n +2 | sort -t',' -k2 -n  | head -n 1 | cut -d',' -f1\`"
 
 BERTREND_BASE_DIR = os.getenv("BERTREND_BASE_DIR", None)
-BASE_PATH = (
-    Path(BERTREND_BASE_DIR)
-    if BERTREND_BASE_DIR
-    else Path(__file__).parent.parent.parent
-)
+BASE_PATH = Path(BERTREND_BASE_DIR) if BERTREND_BASE_DIR else Path.home() / ".bertrend"
 
 # Base dirs
 DATA_PATH = BASE_PATH / "data"

{bertrend-0.3.20 → bertrend-0.3.22}/bertrend/demos/demos_utils/i18n.py

@@ -8,12 +8,10 @@ Provides functionality for translating text between French and English.
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
 
-import streamlit as st
 from typing import Optional
 
-from bertrend_apps.prospective_demo.i18n_translations import (
-    TRANSLATIONS as PROSPECTIVE_TRANSLATIONS,
-)
+import streamlit as st
+
 from bertrend.demos.topic_analysis.i18n_translations import (
     TRANSLATIONS as TOPIC_ANALYSIS_TRANSLATIONS,
 )
@@ -26,9 +24,8 @@ from bertrend.demos.weak_signals.i18n_translations import (
 
 # Merge all translations
 TRANSLATIONS = {
-    **PROSPECTIVE_TRANSLATIONS,
-    **TOPIC_ANALYSIS_TRANSLATIONS,
     **DEMOS_UTILS_TRANSLATIONS,
+    **TOPIC_ANALYSIS_TRANSLATIONS,
     **WEAK_SIGNALS_TRANSLATIONS,
 }
 
@@ -39,12 +36,6 @@ LANGUAGES = {"fr": "Français", "en": "English"}
 DEFAULT_LANGUAGE = "en"
 
 
-def set_default_internationalization_language(language: str) -> None:
-    """Set the default language for the application."""
-    global DEFAULT_LANGUAGE
-    DEFAULT_LANGUAGE = language
-
-
 def get_current_internationalization_language() -> str:
     """Get the currently selected language from the session state."""
     if "internationalization_language" not in st.session_state:
@@ -73,7 +64,7 @@ def create_internationalization_language_selector() -> None:
         st.rerun()
 
 
-def translate(key: str, default: Optional[str] = None) -> str:
+def translate_helper(key: str, default: Optional[str] = None, translations=None) -> str:
     """
     Translate a text key to the current language.
 
@@ -84,12 +75,28 @@ def translate(key: str, default: Optional[str] = None) -> str:
     Returns:
         The translated text in the current language
     """
+    if translations is None:
+        return key
     lang = get_current_internationalization_language()
 
-    if key in TRANSLATIONS and lang in TRANSLATIONS[key]:
-        return TRANSLATIONS[key][lang]
+    if key in translations and lang in translations[key]:
+        return translations[key][lang]
 
     # If the key doesn't exist, or the language is not available for this key
     if default:
         return default
     return key  # Return the key itself as fallback
+
+
+def translate(key: str, default: Optional[str] = None) -> str:
+    """
+    Translate a text key to the current language.
+
+    Args:
+        key: The translation key to look up
+        default: Default text to return if the key is not found
+
+    Returns:
+        The translated text in the current language
+    """
+    return translate_helper(key, default, TRANSLATIONS)

{bertrend-0.3.20 → bertrend-0.3.22}/bertrend/demos/start_summarizer_app.sh

@@ -6,4 +6,4 @@
 #
 
 # Starts the Summarizer app (a comparison of various method of text summarization)
-CUDA_VISIBLE_DEVICES=0 OPENAI_API_KEY=$OPENAI_API_KEY OPENAI_BASE_URL=$OPENAI_BASE_URL OPENAI_DEFAULT_MODEL=$OPENAI_DEFAULT_MODEL streamlit run --theme.primaryColor royalblue summarization/summarizer_app.py
+streamlit run --theme.primaryColor royalblue summarization/summarizer_app.py

bertrend-0.3.22/bertrend/demos/start_topic_analysis_app.sh

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+# Use Python to load .env file and export variables
+eval $(python3 << 'EOF'
+from dotenv import dotenv_values
+from dotenv import find_dotenv
+
+# Find .env file starting from current directory
+dotenv_path = find_dotenv(usecwd=True)
+
+if dotenv_path:
+    # Load only the variables from .env file (not all environment)
+    env_vars = dotenv_values(dotenv_path)
+
+    # Export only the variables from .env
+    for key, value in env_vars.items():
+        if value is not None:  # Skip empty values
+            # Escape special characters in the value
+            value_escaped = value.replace('"', '\\"')
+            print(f'export {key}="{value_escaped}"')
+else:
+    import sys
+    print('echo "Warning: .env file not found"', file=sys.stderr)
+EOF
+)
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Topic Analysis demo"
+cd `pwd`/topic_analysis && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/topic_analysis_demo.log

bertrend-0.3.22/bertrend/demos/start_weak_signals_app.sh

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+# Use Python to load .env file and export variables
+eval $(python3 << 'EOF'
+from dotenv import dotenv_values
+from dotenv import find_dotenv
+
+# Find .env file starting from current directory
+dotenv_path = find_dotenv(usecwd=True)
+
+if dotenv_path:
+    # Load only the variables from .env file (not all environment)
+    env_vars = dotenv_values(dotenv_path)
+
+    # Export only the variables from .env
+    for key, value in env_vars.items():
+        if value is not None:  # Skip empty values
+            # Escape special characters in the value
+            value_escaped = value.replace('"', '\\"')
+            print(f'export {key}="{value_escaped}"')
+else:
+    import sys
+    print('echo "Warning: .env file not found"', file=sys.stderr)
+EOF
+)
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Weak Signal Analysis demo"
+cd `pwd`/weak_signals && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/weak_signals_demo.log

bertrend-0.3.22/bertrend/services/scheduling/.dockerignore

@@ -0,0 +1,52 @@
+# Python
+__pycache__
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+*.sqlite
+*.db
+data/
+logs/
+*.log
+
+# Git
+.git/
+.gitignore
+
+# Docker
+Dockerfile
+docker-compose.yml
+.dockerignore
+
+# Documentation
+README.md
+*.md
+
+# Tests
+tests/
+test_*.py
+*_test.py

bertrend-0.3.22/bertrend/services/scheduling/Docker_setup.md

@@ -0,0 +1,310 @@
+# Job Scheduler - Docker Setup
+
+A FastAPI-based job scheduler with persistent SQLite storage and ProcessPoolExecutor, containerized with Docker.
+
+## 📁 Project Structure
+
+```
+job-scheduler/
+├── main.py                 # FastAPI application
+├── requirements.txt        # Python dependencies
+├── Dockerfile             # Docker image definition
+├── docker-compose.yml     # Docker Compose configuration
+├── .dockerignore          # Docker ignore patterns
+├── data/                  # SQLite database (persistent)
+│   └── jobs.sqlite
+└── logs/                  # Application logs (optional)
+```
+
+## 🚀 Quick Start
+
+### 1. Build and Run
+
+```bash
+# Build and start the container
+docker-compose up -d
+
+# View logs
+docker-compose logs -f
+
+# Check status
+docker-compose ps
+```
+
+### 2. Access the API
+
+- **API Base URL**: http://localhost:8000
+- **API Documentation**: http://localhost:8000/docs
+- **Health Check**: http://localhost:8000/
+
+### 3. Create Your First Job
+
+```bash
+# Create a job that runs every minute
+curl -X POST "http://localhost:8000/jobs" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "job_id": "test_job",
+    "job_type": "interval",
+    "function_name": "sample_job",
+    "minutes": 1,
+    "kwargs": {"message": "Hello from Docker!"}
+  }'
+
+# List all jobs
+curl http://localhost:8000/jobs
+
+# Check job details
+curl http://localhost:8000/jobs/test_job
+```
+
+## 🛠️ Docker Commands
+
+### Basic Operations
+
+```bash
+# Start services
+docker-compose up -d
+
+# Stop services
+docker-compose down
+
+# Restart services
+docker-compose restart
+
+# View logs
+docker-compose logs -f scheduler
+
+# Execute commands in container
+docker-compose exec scheduler bash
+```
+
+### Database Management
+
+```bash
+# Backup database
+docker-compose exec scheduler cp /app/data/jobs.sqlite /app/data/jobs_backup.sqlite
+
+# Access database
+docker-compose exec scheduler sqlite3 /app/data/jobs.sqlite
+```
+
+### Rebuilding
+
+```bash
+# Rebuild after code changes
+docker-compose up -d --build
+
+# Force rebuild
+docker-compose build --no-cache
+docker-compose up -d
+```
+
+## 📊 Monitoring
+
+### Health Check
+
+The container includes automatic health checks:
+
+```bash
+# Check container health
+docker-compose ps
+
+# Manual health check
+curl http://localhost:8000/
+```
+
+### Logs
+
+```bash
+# View real-time logs
+docker-compose logs -f
+
+# View last 100 lines
+docker-compose logs --tail=100
+
+# View logs from specific service
+docker-compose logs scheduler
+```
+
+## 🔧 Configuration
+
+### Environment Variables
+
+Edit `docker-compose.yml` to customize:
+
+```yaml
+environment:
+  - TZ=Europe/Paris          # Timezone
+  - LOG_LEVEL=INFO           # Logging level
+  - MAX_WORKERS=5            # ProcessPoolExecutor workers
+  - DB_PATH=/app/data/jobs.sqlite
+```
+
+### Resource Limits
+
+Adjust CPU and memory limits in `docker-compose.yml`:
+
+```yaml
+deploy:
+  resources:
+    limits:
+      cpus: '2'
+      memory: 1G
+```
+
+### Port Mapping
+
+Change the exposed port:
+
+```yaml
+ports:
+  - "8080:8000"  # Host:Container
+```
+
+## 💾 Data Persistence
+
+The SQLite database is stored in `./data/jobs.sqlite` on your host machine, ensuring jobs persist across container restarts.
+
+### Backup Strategy
+
+```bash
+# Manual backup
+cp data/jobs.sqlite data/jobs_backup_$(date +%Y%m%d).sqlite
+
+# Automated backup script
+#!/bin/bash
+docker-compose exec scheduler sqlite3 /app/data/jobs.sqlite ".backup '/app/data/backup_$(date +%Y%m%d_%H%M%S).sqlite'"
+```
+
+## 🔍 Troubleshooting
+
+### Container won't start
+
+```bash
+# Check logs
+docker-compose logs
+
+# Rebuild
+docker-compose down
+docker-compose up -d --build
+```
+
+### Database locked
+
+```bash
+# Stop all instances
+docker-compose down
+
+# Remove stale locks
+rm data/jobs.sqlite-journal
+
+# Restart
+docker-compose up -d
+```
+
+### Permission issues
+
+```bash
+# Fix permissions
+sudo chown -R $USER:$USER data/
+chmod -R 755 data/
+```
+
+## 🚢 Production Deployment
+
+### Using Docker Swarm
+
+```bash
+# Initialize swarm
+docker swarm init
+
+# Deploy stack
+docker stack deploy -c docker-compose.yml scheduler
+
+# Check services
+docker stack services scheduler
+```
+
+### Using Kubernetes
+
+Create `deployment.yaml`:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: job-scheduler
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: scheduler
+  template:
+    metadata:
+      labels:
+        app: scheduler
+    spec:
+      containers:
+      - name: scheduler
+        image: job-scheduler:latest
+        ports:
+        - containerPort: 8000
+        volumeMounts:
+        - name: data
+          mountPath: /app/data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: scheduler-data
+```
+
+## 📝 API Examples
+
+### Cron Job (Daily at 9 AM Paris time)
+
+```bash
+curl -X POST "http://localhost:8000/jobs" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "job_id": "daily_report",
+    "job_type": "cron",
+    "function_name": "report_generator",
+    "cron_expression": "0 9 * * *",
+    "kwargs": {"report_type": "daily"}
+  }'
+```
+
+### Validate Cron Expression
+
+```bash
+curl -X POST "http://localhost:8000/cron/validate" \
+  -H "Content-Type: application/json" \
+  -d '{"expression": "*/15 * * * *"}'
+```
+
+### Get Cron Examples
+
+```bash
+curl http://localhost:8000/cron/examples
+```
+
+## 🔐 Security Considerations
+
+For production:
+
+1. **Add authentication**: Implement API key or OAuth
+2. **Use HTTPS**: Add reverse proxy (nginx/traefik)
+3. **Limit exposure**: Use internal networks
+4. **Regular backups**: Automated database backups
+5. **Update dependencies**: Keep packages current
+
+## 📚 Additional Resources
+
+- [FastAPI Documentation](https://fastapi.tiangolo.com/)
+- [APScheduler Documentation](https://apscheduler.readthedocs.io/)
+- [Docker Documentation](https://docs.docker.com/)
+
+## 📄 License
+
+This project is provided as-is for educational and commercial use.

bertrend-0.3.22/bertrend/services/scheduling/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.12-slim
+
+# Set working directory
+WORKDIR /app
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+# Copy requirements file
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . /app/bertrend/services/scheduling
+
+# Expose port
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=60s --timeout=10s --start-period=5s --retries=3 \
+    CMD python -c "import requests; requests.get('http://localhost:8000/')" || exit 1
+
+# Run the application
+CMD ["uvicorn", "bertrend.services.scheduling.scheduling_service:app", "--host", "0.0.0.0", "--port", "8000"]