bertrend 0.4.7__tar.gz → 0.4.9__tar.gz

{bertrend-0.4.7 → bertrend-0.4.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.7
+Version: 0.4.9
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -448,6 +448,11 @@ Provides various statistics about topics:
 
 ![llm_interpretation](docs/images/llm_interpretation.png)
 
+## Article Scoring Service
+
+For service setup and API details, see:
+[`docs/services/article_scoring_service.md`](docs/services/article_scoring_service.md).
+
 ## Support
 
 Please use the 'Issues' section to report and discuss any problems with BERTrend.

{bertrend-0.4.7 → bertrend-0.4.9}/README.md

@@ -364,6 +364,11 @@ Provides various statistics about topics:
 
 ![llm_interpretation](docs/images/llm_interpretation.png)
 
+## Article Scoring Service
+
+For service setup and API details, see:
+[`docs/services/article_scoring_service.md`](docs/services/article_scoring_service.md).
+
 ## Support
 
 Please use the 'Issues' section to report and discuss any problems with BERTrend.

{bertrend-0.4.7 → bertrend-0.4.9}/bertrend/bertrend_apps/data_provider/data_provider.py

@@ -2,7 +2,6 @@
 #  See AUTHORS.txt
 #  SPDX-License-Identifier: MPL-2.0
 #  This file is part of BERTrend.
-import asyncio
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
@@ -10,6 +9,7 @@ from pathlib import Path
 import jsonlines
 import langdetect
 import pandas as pd
+import requests
 from dateutil import parser
 from goose3 import Goose
 from joblib import Parallel, delayed
@@ -17,12 +17,11 @@ from loguru import logger
 from newspaper import Article
 
 from bertrend.article_scoring.article_scoring import QualityLevel
-from bertrend.article_scoring.scoring_agent import score_articles
-from bertrend.utils.data_loading import TEXT_COLUMN
 from bertrend.bertrend_apps.data_provider.utils import (
     decode_google_news_url,
     wait_if_seen_url,
 )
+from bertrend.utils.data_loading import TEXT_COLUMN
 
 # Ensures to write with +rw for both user and groups
 os.umask(0o002)
@@ -85,27 +84,53 @@ class DataProvider(ABC):
 
     @staticmethod
     def evaluate_quality(
-        articles: list[dict], minimum_quality_level: QualityLevel
+        articles: list[dict], minimum_quality_level: QualityLevel = QualityLevel.AVERAGE
     ) -> list[dict]:
         texts = [article[TEXT_COLUMN] for article in articles]
+        article_scoring_service_url = os.getenv("ARTICLE_SCORING_SERVICE_URL")
+
+        if not article_scoring_service_url:
+            logger.warning(
+                "ARTICLE_SCORING_SERVICE_URL is not set; skipping article quality filtering."
+            )
+            return articles
+
+        try:
+            response = requests.post(
+                f"{article_scoring_service_url.rstrip('/')}/score",
+                json={"articles": texts},
+                timeout=300,
+            )
+            response.raise_for_status()
+            payload = response.json()
+            results = payload.get("results", [])
+        except requests.RequestException as exc:
+            logger.warning(
+                f"Article scoring service unavailable at {article_scoring_service_url}: {exc}. "
+                "Skipping article quality filtering."
+            )
+            return articles
 
-        # Evaluate quality of articles using a LLM agent
-        results = asyncio.run(score_articles(texts))
+        if len(results) != len(texts):
+            logger.warning(
+                "Article scoring service returned an unexpected number of results; "
+                "skipping article quality filtering."
+            )
+            return articles
 
-        # Filter out articles with quality < minimum_quality
+        # Filter out articles with quality < minimum_quality_level
         filtered_articles = [
             {
                 **article,
-                "quality_metrics": results[i].output.model_dump(),
-                "overall_quality": results[i].output.quality_level.name,
+                "quality_metrics": results[i]["quality_metrics"],
+                "overall_quality": results[i]["overall_quality"],
             }
-            # 1. Ensure output is NOT None before proceeding
             for i, article in enumerate(articles)
-            if results[i].output is not None
-            # 2. Then, check the quality level
-            and results[i].output.quality_level >= minimum_quality_level
+            if results[i].get("overall_quality") in QualityLevel.__members__
+            and QualityLevel[results[i]["overall_quality"]] >= minimum_quality_level
+            and not results[i].get("error")
         ]
-        assert len(results) == len(texts)
+
         logger.info(
             f"Filtered out {len(articles) - len(filtered_articles)} articles with quality < {minimum_quality_level.name}"
         )

{bertrend-0.4.7 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_management/bertrend_worker.py

@@ -253,6 +253,45 @@ class BertrendWorker:
                 "traceback": traceback.format_exc(),
             }
 
+    def _get_retry_count(self, message: aio_pika.abc.AbstractIncomingMessage) -> int:
+        """Return the current retry count stored in the message header x-retry-count."""
+        headers = message.headers or {}
+        try:
+            return int(headers.get("x-retry-count", 0))
+        except (TypeError, ValueError):
+            return 0
+
+    async def _nack_or_discard(
+        self,
+        message: aio_pika.abc.AbstractIncomingMessage,
+        reason: str,
+        correlation_id: str | None,
+    ):
+        """
+        Retry the message if the retry limit has not been reached, otherwise
+        reject it (requeue=False) so it goes to the DLQ.
+
+        Delegates the retry-limit check to republish_with_retry(), which
+        returns False when the limit is reached (and skips publishing).
+        The original message is always rejected with requeue=False so that
+        RabbitMQ does not re-deliver the same copy.
+        """
+        retry_count = self._get_retry_count(message)
+        republished = await self.queue_manager.republish_with_retry(
+            message, retry_count
+        )
+        if republished:
+            logger.warning(
+                f"Message {correlation_id} failed (attempt {retry_count + 1}/{self.config.max_retries}) "
+                f"— republished with incremented retry count. Reason: {reason}"
+            )
+        else:
+            logger.warning(
+                f"Message {correlation_id} exceeded max retries ({self.config.max_retries}) "
+                f"— sending to DLQ. Reason: {reason}"
+            )
+        await message.reject(requeue=False)
+
     async def callback(
         self,
         message: aio_pika.abc.AbstractIncomingMessage,
@@ -268,41 +307,63 @@ class BertrendWorker:
 
             logger.info(f"Request endpoint: {request_data.get('endpoint')}")
 
-            # Process request
-            response_data = await self.process_request(request_data)
+            # Process request with a timeout to avoid blocking the queue indefinitely
+            response_data = await asyncio.wait_for(
+                self.process_request(request_data),
+                timeout=self.config.job_timeout,
+            )
 
             # Add metadata
             response_data["correlation_id"] = correlation_id
             response_data["request_data"] = request_data
 
-            # Publish response/error if reply_to is specified
-            if message.reply_to:
-                if response_data.get("status") == "error":
-                    await self.queue_manager.publish_error(
-                        error_data=response_data, correlation_id=correlation_id
-                    )
-                else:
-                    await self.queue_manager.publish_response(
-                        response_data=response_data, correlation_id=correlation_id
-                    )
-
-            # Acknowledge message
-            await message.ack()
-            logger.info(
-                f"Completed request: {correlation_id} - Status: {response_data.get('status')}"
-            )
-
         except json.JSONDecodeError as e:
             logger.error(f"Invalid message format: {str(e)}")
             # Reject and don't requeue invalid messages
             await message.reject(requeue=False)
+            return
+
+        except asyncio.TimeoutError:
+            await self._nack_or_discard(
+                message,
+                reason=f"job timed out after {self.config.job_timeout}s",
+                correlation_id=correlation_id,
+            )
+            return
 
         except Exception as e:
             logger.error(f"Error processing message: {str(e)}")
             logger.error(traceback.format_exc())
+            await self._nack_or_discard(
+                message,
+                reason=str(e),
+                correlation_id=correlation_id,
+            )
+            return
 
-            # Requeue for retry (will go to DLQ after max retries)
-            await message.nack(requeue=True)
+        # Acknowledge message before publishing the response, so that the message
+        # is always acked regardless of any subsequent publish failure
+        await message.ack()
+        logger.info(
+            f"Completed request: {correlation_id} - Status: {response_data.get('status')}"
+        )
+
+        # Publish response/error if reply_to is specified
+        if message.reply_to:
+            try:
+                if response_data.get("status") == "error":
+                    await self.queue_manager.publish_error(
+                        error_data=response_data, correlation_id=correlation_id
+                    )
+                else:
+                    await self.queue_manager.publish_response(
+                        response_data=response_data, correlation_id=correlation_id
+                    )
+            except Exception as e:
+                logger.error(
+                    f"Failed to publish response for {correlation_id}: {str(e)}"
+                )
+                logger.error(traceback.format_exc())
 
     async def start(self):
         """Start the worker"""

{bertrend-0.4.7 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_management/queue_manager.py

@@ -159,6 +159,55 @@ class QueueManager:
 
         await queue.consume(callback, no_ack=False)
 
+    async def republish_with_retry(
+        self,
+        original_message: aio_pika.abc.AbstractIncomingMessage,
+        retry_count: int,
+    ) -> bool:
+        """Republish a failed message with an incremented x-retry-count header.
+
+        RabbitMQ does not modify headers on requeue, so we must republish
+        manually with the updated counter instead of using nack(requeue=True).
+
+        Returns True if the message was republished, False if the retry limit
+        has already been reached and the message was not republished.
+        """
+        new_retry_count = retry_count + 1
+        if new_retry_count > self.config.max_retries:
+            logger.warning(
+                f"Message {original_message.correlation_id} has reached the maximum "
+                f"retry limit ({self.config.max_retries}) — not republishing."
+            )
+            return False
+
+        if not self.channel or self.channel.is_closed:
+            await self.connect()
+
+        # Carry over all existing headers and bump the retry counter
+        headers = dict(original_message.headers or {})
+        headers["x-retry-count"] = new_retry_count
+
+        message = aio_pika.Message(
+            body=original_message.body,
+            headers=headers,
+            delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
+            priority=original_message.priority,
+            correlation_id=original_message.correlation_id,
+            content_type=original_message.content_type,
+            reply_to=original_message.reply_to,
+        )
+
+        await self.channel.default_exchange.publish(
+            message,
+            routing_key=self.config.request_queue,
+        )
+
+        logger.info(
+            f"Republished message {original_message.correlation_id} "
+            f"with x-retry-count={new_retry_count}"
+        )
+        return True
+
     async def publish_response(self, response_data: dict, correlation_id: str):
         """Publish response to response queue_management"""
         if not self.channel or self.channel.is_closed:

{bertrend-0.4.7 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_management/rabbitmq_config.py

@@ -36,5 +36,8 @@ class RabbitMQConfig:
     prefetch_count: int = int(os.getenv("RABBITMQ_PREFETCH_COUNT", 1))
 
     # Retry configuration
-    max_retries: int = int(os.getenv("RABBITMQ_MAX_RETRIES", 3))
+    max_retries: int = int(os.getenv("RABBITMQ_MAX_RETRIES", 2))
     retry_delay: int = int(os.getenv("RABBITMQ_RETRY_DELAY", 5000))  # milliseconds
+
+    # Job timeout: max seconds a single job may run before being nacked (default: 1 hour)
+    job_timeout: int = int(os.getenv("RABBITMQ_JOB_TIMEOUT", 3600))

{bertrend-0.4.7 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_monitor/app.py

@@ -60,6 +60,18 @@ class RabbitMQAdminClient:
         r.raise_for_status()
         return r.json()
 
+    def get_consumers(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/api/consumers/{self._vhost_path()}"
+        r = self._session.get(url, timeout=5)
+        r.raise_for_status()
+        return r.json()
+
+    def get_channels(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/api/channels"
+        r = self._session.get(url, timeout=5)
+        r.raise_for_status()
+        return r.json()
+
     def peek_messages(self, name: str, count: int = 10) -> List[Dict[str, Any]]:
         url = f"{self.base_url}/api/queues/{self._vhost_path()}/{quote(name, safe='')}/get"
         payload = {
@@ -262,7 +274,14 @@ client = RabbitMQAdminClient(client_cfg, management_url=mgmt_url)
 
 # --- Data Fetching ---
 queue_data = {}
-global_stats = {"total": 0, "ready": 0, "unacked": 0, "consumers": 0}
+global_stats = {
+    "total": 0,
+    "requests": 0,
+    "responses": 0,
+    "failed": 0,
+    "unacked": 0,
+    "consumers": 0,
+}
 error_msg = None
 
 try:
@@ -272,9 +291,14 @@ try:
 
         # Aggregation
         global_stats["total"] += info.get("messages", 0)
-        global_stats["ready"] += info.get("messages_ready", 0)
         global_stats["unacked"] += info.get("messages_unacknowledged", 0)
         global_stats["consumers"] += info.get("consumers", 0)
+        if "requests" in q:
+            global_stats["requests"] += info.get("messages", 0)
+        elif "responses" in q or "response" in q:
+            global_stats["responses"] += info.get("messages", 0)
+        elif "failed" in q:
+            global_stats["failed"] += info.get("messages", 0)
 
 except Exception as e:
     error_msg = str(e)
@@ -288,18 +312,33 @@ if error_msg:
 
 # 1. Global Overview Row
 st.markdown(f"### {DASHBOARD_ICON} System Overview")
-kpi1, kpi2, kpi3, kpi4 = st.columns(4)
+kpi1, kpi2, kpi3, kpi4, kpi5, kpi6 = st.columns(6)
 kpi1.metric(
     "Total Backlog", global_stats["total"], help="Total messages in all selected queues"
 )
-kpi2.metric("Ready to Process", global_stats["ready"], delta_color="normal")
+kpi2.metric(
+    "Requests",
+    global_stats["requests"],
+    help="Messages in bertrend_requests queue",
+)
 kpi3.metric(
+    "Responses",
+    global_stats["responses"],
+    help="Messages in bertrend_responses queue",
+)
+kpi4.metric(
+    "Failed",
+    global_stats["failed"],
+    delta_color="inverse",
+    help="Messages in bertrend_failed queue",
+)
+kpi5.metric(
     "Unacknowledged",
     global_stats["unacked"],
     delta_color="inverse",
     help="Messages currently being processed but not acked",
 )
-kpi4.metric("Active Consumers", global_stats["consumers"])
+kpi6.metric("Active Consumers", global_stats["consumers"])
 
 st.divider()
 
@@ -327,6 +366,108 @@ else:
 
             st.divider()
 
+            # B. Unacknowledged Messages Details
+            unacked_count = info.get("messages_unacknowledged", 0)
+            with st.expander(
+                f"Unacknowledged Messages ({unacked_count})",
+                expanded=unacked_count > 0,
+            ):
+                if unacked_count == 0:
+                    st.info("No unacknowledged messages in this queue.", icon=INFO_ICON)
+                else:
+                    try:
+                        all_consumers = client.get_consumers()
+                        all_channels = client.get_channels()
+
+                        # Filter consumers for this queue
+                        queue_consumers = [
+                            c
+                            for c in all_consumers
+                            if c.get("queue", {}).get("name") == qname
+                        ]
+
+                        # Build a map of channel_name -> channel details
+                        channel_map = {ch.get("name"): ch for ch in all_channels}
+
+                        if not queue_consumers:
+                            st.warning(
+                                "Unacknowledged messages exist but no active consumers found. "
+                                "Messages may be held by a disconnected consumer."
+                            )
+                        else:
+                            st.caption(
+                                f"{len(queue_consumers)} consumer(s) holding unacknowledged messages:"
+                            )
+                            for cons in queue_consumers:
+                                ch_name = cons.get("channel_details", {}).get(
+                                    "name", ""
+                                )
+                                ch = channel_map.get(ch_name, {})
+                                ch_unacked = ch.get("messages_unacknowledged", 0)
+                                consumer_tag = cons.get("consumer_tag", "unknown")
+                                peer_host = cons.get("channel_details", {}).get(
+                                    "peer_host", "?"
+                                )
+                                peer_port = cons.get("channel_details", {}).get(
+                                    "peer_port", ""
+                                )
+                                prefetch = cons.get("prefetch_count", 0)
+                                ack_required = cons.get("ack_required", True)
+
+                                with st.container(border=True):
+                                    cc1, cc2, cc3, cc4 = st.columns(4)
+                                    cc1.metric("Unacked (channel)", ch_unacked)
+                                    cc2.markdown(f"**Consumer Tag**\n`{consumer_tag}`")
+                                    cc3.markdown(
+                                        f"**Client**\n`{peer_host}:{peer_port}`"
+                                    )
+                                    cc4.markdown(
+                                        f"**Prefetch / Ack required**\n`{prefetch}` / `{ack_required}`"
+                                    )
+
+                                    # Channel-level stats
+                                    ch_stats = ch.get("message_stats", {})
+                                    ack_rate = (
+                                        ch_stats.get("ack_details", {}).get("rate", 0.0)
+                                        if ch_stats
+                                        else 0.0
+                                    )
+                                    deliver_rate = (
+                                        ch_stats.get("deliver_details", {}).get(
+                                            "rate", 0.0
+                                        )
+                                        if ch_stats
+                                        else 0.0
+                                    )
+                                    total_acked = (
+                                        ch_stats.get("ack", 0) if ch_stats else 0
+                                    )
+                                    total_delivered = (
+                                        ch_stats.get("deliver", 0) if ch_stats else 0
+                                    )
+
+                                    sc1, sc2, sc3, sc4 = st.columns(4)
+                                    sc1.metric("Ack Rate", f"{ack_rate:.2f}/s")
+                                    sc2.metric("Deliver Rate", f"{deliver_rate:.2f}/s")
+                                    sc3.metric("Total Acked", total_acked)
+                                    sc4.metric("Total Delivered", total_delivered)
+
+                                    # Connection info
+                                    conn = ch.get("connection_details", {})
+                                    if conn:
+                                        st.caption(
+                                            f"Connection: `{conn.get('name', ch_name)}` "
+                                            f"— state: `{ch.get('state', '?')}`"
+                                        )
+
+                    except Exception as e:
+                        st.error(
+                            f"Failed to fetch consumer/channel details: {e}",
+                            icon=ERROR_ICON,
+                        )
+
+            st.divider()
+
             # B. Message Peeking Interface
             c_head, c_peek_n, c_sort = st.columns([3, 1, 1])
             c_head.subheader("Message Browser")
@@ -546,6 +687,7 @@ else:
                     obj = d["obj"]
 
                     props = m.get("properties", {})
+                    headers = props.get("headers") or {}
                     corr_id = props.get("correlation_id", "No-ID")
                     msg_size = (
                         m.get("payload_bytes", 0)
@@ -562,6 +704,9 @@ else:
                         info_tags.append(f"User: {d['user']}")
                     if d["model_id"]:
                         info_tags.append(f"Model: {d['model_id']}")
+                    retry_count = headers.get("x-retry-count")
+                    if retry_count is not None:
+                        info_tags.append(f"Retry: {retry_count}")
 
                     info_str = f" | {' | '.join(info_tags)}" if info_tags else ""
                     label = f"#{idx + 1} | {fmt} | CID: {corr_id}{info_str} | {msg_size} bytes"
@@ -617,6 +762,12 @@ else:
                                             "Key": "App ID",
                                             "Val": str(props.get("app_id") or ""),
                                         },
+                                        {
+                                            "Key": "x-retry-count",
+                                            "Val": str(headers.get("x-retry-count"))
+                                            if headers.get("x-retry-count") is not None
+                                            else "",
+                                        },
                                     ]
                                 ).dropna()
                                 st.dataframe(meta_df, hide_index=True, width="stretch")

bertrend-0.4.9/bertrend/services/article_scoring_server/config/default_config.toml

@@ -0,0 +1,3 @@
+host = "0.0.0.0"
+port = 8887
+number_workers = 1

bertrend-0.4.9/bertrend/services/article_scoring_server/config/settings.py

@@ -0,0 +1,32 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import os
+import tomllib
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+DEFAULT_CONFIG_FILE = Path(__file__).parent / "default_config.toml"
+
+
+class ArticleScoringAPIConfig(BaseSettings):
+    host: str
+    port: int
+    number_workers: int
+
+
+def get_config() -> ArticleScoringAPIConfig:
+    """
+    Return the configuration for the summary API.
+    The configuration is loaded from a toml file.
+    The path to the toml file can be specified using the environment variable `SUMMARY_API_CONFIG_FILE`.
+    If the environment variable is not set, the default configuration file is used.
+    """
+    config_file = os.environ.get("ARTICLE_SCORING_API_CONFIG_FILE", None)
+    if config_file is None:
+        config_file = DEFAULT_CONFIG_FILE
+    with open(config_file, "rb") as f:
+        return ArticleScoringAPIConfig(**tomllib.load(f))

bertrend-0.4.9/bertrend/services/article_scoring_server/models.py

@@ -0,0 +1,21 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from pydantic import BaseModel
+
+
+class ScoreArticlesRequest(BaseModel):
+    articles: list[str]
+
+
+class ScoredArticle(BaseModel):
+    article_index: int
+    quality_metrics: dict | None = None
+    overall_quality: str | None = None
+    error: str | None = None
+
+
+class ScoreArticlesResponse(BaseModel):
+    results: list[ScoredArticle]

bertrend-0.4.9/bertrend/services/article_scoring_server/routers/info.py

@@ -0,0 +1,19 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter
+from starlette.responses import RedirectResponse
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@router.get("/", summary="Redirect to API documentation")
+def root():
+    return RedirectResponse(url="/docs")

bertrend-0.4.9/bertrend/services/article_scoring_server/routers/scoring.py

@@ -0,0 +1,43 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter, HTTPException
+
+from bertrend.article_scoring.scoring_agent import score_articles
+from bertrend.services.article_scoring_server.models import (
+    ScoreArticlesRequest,
+    ScoreArticlesResponse,
+    ScoredArticle,
+)
+
+router = APIRouter()
+
+
+@router.post("/score", response_model=ScoreArticlesResponse)
+async def score(request: ScoreArticlesRequest):
+    try:
+        score_results = await score_articles(request.articles)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to score articles: {e}",
+        ) from e
+
+    results = []
+    for index, item in enumerate(score_results):
+        quality_metrics = item.output.serialize_model() if item.output else None
+        overall_quality = (
+            item.output.quality_level.name if item.output and not item.error else None
+        )
+        results.append(
+            ScoredArticle(
+                article_index=index,
+                quality_metrics=quality_metrics,
+                overall_quality=overall_quality,
+                error=item.error,
+            )
+        )
+
+    return ScoreArticlesResponse(results=results)