bertrend 0.4.7__tar.gz → 0.4.8__tar.gz

{bertrend-0.4.7 → bertrend-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.7
+Version: 0.4.8
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -448,6 +448,11 @@ Provides various statistics about topics:
 
 ![llm_interpretation](docs/images/llm_interpretation.png)
 
+## Article Scoring Service
+
+For service setup and API details, see:
+[`docs/services/article_scoring_service.md`](docs/services/article_scoring_service.md).
+
 ## Support
 
 Please use the 'Issues' section to report and discuss any problems with BERTrend.

{bertrend-0.4.7 → bertrend-0.4.8}/README.md

@@ -364,6 +364,11 @@ Provides various statistics about topics:
 
 ![llm_interpretation](docs/images/llm_interpretation.png)
 
+## Article Scoring Service
+
+For service setup and API details, see:
+[`docs/services/article_scoring_service.md`](docs/services/article_scoring_service.md).
+
 ## Support
 
 Please use the 'Issues' section to report and discuss any problems with BERTrend.

{bertrend-0.4.7 → bertrend-0.4.8}/bertrend/bertrend_apps/data_provider/data_provider.py

@@ -2,7 +2,6 @@
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
-import asyncio
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
@@ -10,6 +9,7 @@ from pathlib import Path
 import jsonlines
 import langdetect
 import pandas as pd
+import requests
 from dateutil import parser
 from goose3 import Goose
 from joblib import Parallel, delayed
@@ -17,12 +17,11 @@ from loguru import logger
 from newspaper import Article
 
 from bertrend.article_scoring.article_scoring import QualityLevel
-from bertrend.article_scoring.scoring_agent import score_articles
-from bertrend.utils.data_loading import TEXT_COLUMN
 from bertrend.bertrend_apps.data_provider.utils import (
     decode_google_news_url,
     wait_if_seen_url,
 )
+from bertrend.utils.data_loading import TEXT_COLUMN
 
 # Ensures to write with +rw for both user and groups
 os.umask(0o002)
@@ -85,27 +84,53 @@ class DataProvider(ABC):
 
     @staticmethod
     def evaluate_quality(
-        articles: list[dict], minimum_quality_level: QualityLevel
+        articles: list[dict], minimum_quality_level: QualityLevel = QualityLevel.AVERAGE
     ) -> list[dict]:
         texts = [article[TEXT_COLUMN] for article in articles]
+        article_scoring_service_url = os.getenv("ARTICLE_SCORING_SERVICE_URL")
+
+        if not article_scoring_service_url:
+            logger.warning(
+                "ARTICLE_SCORING_SERVICE_URL is not set; skipping article quality filtering."
+            )
+            return articles
+
+        try:
+            response = requests.post(
+                f"{article_scoring_service_url.rstrip('/')}/score",
+                json={"articles": texts},
+                timeout=300,
+            )
+            response.raise_for_status()
+            payload = response.json()
+            results = payload.get("results", [])
+        except requests.RequestException as exc:
+            logger.warning(
+                f"Article scoring service unavailable at {article_scoring_service_url}: {exc}. "
+                "Skipping article quality filtering."
+            )
+            return articles
 
-        # Evaluate quality of articles using a LLM agent
-        results = asyncio.run(score_articles(texts))
+        if len(results) != len(texts):
+            logger.warning(
+                "Article scoring service returned an unexpected number of results; "
+                "skipping article quality filtering."
+            )
+            return articles
 
-        # Filter out articles with quality < minimum_quality
+        # Filter out articles with quality < minimum_quality_level
         filtered_articles = [
             {
                 **article,
-                "quality_metrics": results[i].output.model_dump(),
-                "overall_quality": results[i].output.quality_level.name,
+                "quality_metrics": results[i]["quality_metrics"],
+                "overall_quality": results[i]["overall_quality"],
             }
-            # 1. Ensure output is NOT None before proceeding
             for i, article in enumerate(articles)
-            if results[i].output is not None
-            # 2. Then, check the quality level
-            and results[i].output.quality_level >= minimum_quality_level
+            if results[i].get("overall_quality") in QualityLevel.__members__
+            and QualityLevel[results[i]["overall_quality"]] >= minimum_quality_level
+            and not results[i].get("error")
         ]
-        assert len(results) == len(texts)
+
         logger.info(
             f"Filtered out {len(articles) - len(filtered_articles)} articles with quality < {minimum_quality_level.name}"
         )

bertrend-0.4.8/bertrend/services/article_scoring_server/config/default_config.toml

@@ -0,0 +1,3 @@
+host = "0.0.0.0"
+port = 8887
+number_workers = 1

bertrend-0.4.8/bertrend/services/article_scoring_server/config/settings.py

@@ -0,0 +1,32 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import os
+import tomllib
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+DEFAULT_CONFIG_FILE = Path(__file__).parent / "default_config.toml"
+
+
+class ArticleScoringAPIConfig(BaseSettings):
+    host: str
+    port: int
+    number_workers: int
+
+
+def get_config() -> ArticleScoringAPIConfig:
+    """
+    Return the configuration for the summary API.
+    The configuration is loaded from a toml file.
+    The path to the toml file can be specified using the environment variable `SUMMARY_API_CONFIG_FILE`.
+    If the environment variable is not set, the default configuration file is used.
+    """
+    config_file = os.environ.get("ARTICLE_SCORING_API_CONFIG_FILE", None)
+    if config_file is None:
+        config_file = DEFAULT_CONFIG_FILE
+    with open(config_file, "rb") as f:
+        return ArticleScoringAPIConfig(**tomllib.load(f))

bertrend-0.4.8/bertrend/services/article_scoring_server/models.py

@@ -0,0 +1,21 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from pydantic import BaseModel
+
+
+class ScoreArticlesRequest(BaseModel):
+    articles: list[str]
+
+
+class ScoredArticle(BaseModel):
+    article_index: int
+    quality_metrics: dict | None = None
+    overall_quality: str | None = None
+    error: str | None = None
+
+
+class ScoreArticlesResponse(BaseModel):
+    results: list[ScoredArticle]

bertrend-0.4.8/bertrend/services/article_scoring_server/routers/info.py

@@ -0,0 +1,19 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter
+from starlette.responses import RedirectResponse
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@router.get("/", summary="Redirect to API documentation")
+def root():
+    return RedirectResponse(url="/docs")

bertrend-0.4.8/bertrend/services/article_scoring_server/routers/scoring.py

@@ -0,0 +1,43 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter, HTTPException
+
+from bertrend.article_scoring.scoring_agent import score_articles
+from bertrend.services.article_scoring_server.models import (
+    ScoreArticlesRequest,
+    ScoreArticlesResponse,
+    ScoredArticle,
+)
+
+router = APIRouter()
+
+
+@router.post("/score", response_model=ScoreArticlesResponse)
+async def score(request: ScoreArticlesRequest):
+    try:
+        score_results = await score_articles(request.articles)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to score articles: {e}",
+        ) from e
+
+    results = []
+    for index, item in enumerate(score_results):
+        quality_metrics = item.output.serialize_model() if item.output else None
+        overall_quality = (
+            item.output.quality_level.name if item.output and not item.error else None
+        )
+        results.append(
+            ScoredArticle(
+                article_index=index,
+                quality_metrics=quality_metrics,
+                overall_quality=overall_quality,
+                error=item.error,
+            )
+        )
+
+    return ScoreArticlesResponse(results=results)

bertrend-0.4.8/bertrend/services/article_scoring_server/scoring_service.py

@@ -0,0 +1,12 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import FastAPI
+
+from bertrend.services.article_scoring_server.routers import info, scoring
+
+app = FastAPI(title="BERTrend Article Scoring Service")
+app.include_router(info.router, tags=["Info"])
+app.include_router(scoring.router, tags=["Article scoring"])

bertrend-0.4.8/bertrend/services/article_scoring_server/start.py

@@ -0,0 +1,18 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import uvicorn
+
+from bertrend.services.article_scoring_server.config.settings import get_config
+
+CONFIG = get_config()
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "bertrend.services.article_scoring_server.scoring_service:app",
+        host=CONFIG.host,
+        port=CONFIG.port,
+        workers=CONFIG.number_workers,
+    )

{bertrend-0.4.7 → bertrend-0.4.8}/bertrend/tests/data_provider/test_data_provider_base.py

@@ -6,6 +6,7 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+import requests
 
 from bertrend.article_scoring.article_scoring import QualityLevel
 from bertrend.bertrend_apps.data_provider.data_provider import DataProvider
@@ -80,23 +81,34 @@ def test_get_text(mock_goose, provider):
     assert title == "Goose title"
 
 
-@patch("bertrend.bertrend_apps.data_provider.data_provider.asyncio.run")
-def test_evaluate_quality(mock_asyncio_run, provider):
+@patch.dict(
+    "os.environ", {"ARTICLE_SCORING_SERVICE_URL": "http://localhost:8887"}, clear=False
+)
+@patch("bertrend.bertrend_apps.data_provider.data_provider.requests.post")
+def test_evaluate_quality(mock_requests_post, provider):
     articles = [
         {"text": "high quality content", "id": 1},
         {"text": "low quality", "id": 2},
     ]
 
-    # Mock score_articles result
-    mock_result_high = MagicMock()
-    mock_result_high.output.quality_level = QualityLevel.EXCELLENT
-    mock_result_high.output.model_dump.return_value = {"score": 0.9}
-
-    mock_result_low = MagicMock()
-    mock_result_low.output.quality_level = QualityLevel.POOR
-    mock_result_low.output.model_dump.return_value = {"score": 0.1}
-
-    mock_asyncio_run.return_value = [mock_result_high, mock_result_low]
+    mock_response = MagicMock()
+    mock_response.json.return_value = {
+        "results": [
+            {
+                "article_index": 0,
+                "quality_metrics": {"score": 0.9},
+                "overall_quality": "EXCELLENT",
+                "error": None,
+            },
+            {
+                "article_index": 1,
+                "quality_metrics": {"score": 0.1},
+                "overall_quality": "POOR",
+                "error": None,
+            },
+        ]
+    }
+    mock_requests_post.return_value = mock_response
 
     filtered = provider.evaluate_quality(articles, QualityLevel.AVERAGE)
 
@@ -105,6 +117,22 @@ def test_evaluate_quality(mock_asyncio_run, provider):
     assert filtered[0]["overall_quality"] == "EXCELLENT"
 
 
+@patch.dict(
+    "os.environ", {"ARTICLE_SCORING_SERVICE_URL": "http://localhost:8887"}, clear=False
+)
+@patch("bertrend.bertrend_apps.data_provider.data_provider.requests.post")
+def test_evaluate_quality_service_unavailable(mock_requests_post, provider):
+    articles = [
+        {"text": "high quality content", "id": 1},
+        {"text": "low quality", "id": 2},
+    ]
+    mock_requests_post.side_effect = requests.ConnectionError("service unavailable")
+
+    filtered = provider.evaluate_quality(articles, QualityLevel.AVERAGE)
+
+    assert filtered == articles
+
+
 def test_process_entries(provider):
     entries = [{"title": "Entry 1"}, {"title": "Entry 2"}]
 

bertrend-0.4.8/bertrend/tests/services/summarizer/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.

bertrend-0.4.8/bertrend/tests/services/summary/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.

bertrend-0.4.8/bertrend/tests/utils/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.

{bertrend-0.4.7 → bertrend-0.4.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bertrend"
-version = "0.4.7"
+version = "0.4.8"
 description = "BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM"
 license = "LICENSE.md"
 readme = "README.md"