bertrend 0.4.8__tar.gz → 0.4.9__tar.gz

{bertrend-0.4.8 → bertrend-0.4.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.8
+Version: 0.4.9
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt

{bertrend-0.4.8 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_management/bertrend_worker.py

@@ -253,6 +253,45 @@ class BertrendWorker:
                 "traceback": traceback.format_exc(),
             }
 
+    def _get_retry_count(self, message: aio_pika.abc.AbstractIncomingMessage) -> int:
+        """Return the current retry count stored in the message header x-retry-count."""
+        headers = message.headers or {}
+        try:
+            return int(headers.get("x-retry-count", 0))
+        except (TypeError, ValueError):
+            return 0
+
+    async def _nack_or_discard(
+        self,
+        message: aio_pika.abc.AbstractIncomingMessage,
+        reason: str,
+        correlation_id: str | None,
+    ):
+        """
+        Retry the message if the retry limit has not been reached, otherwise
+        reject it (requeue=False) so it goes to the DLQ.
+
+        Delegates the retry-limit check to republish_with_retry(), which
+        returns False when the limit is reached (and skips publishing).
+        The original message is always rejected with requeue=False so that
+        RabbitMQ does not re-deliver the same copy.
+        """
+        retry_count = self._get_retry_count(message)
+        republished = await self.queue_manager.republish_with_retry(
+            message, retry_count
+        )
+        if republished:
+            logger.warning(
+                f"Message {correlation_id} failed (attempt {retry_count + 1}/{self.config.max_retries}) "
+                f"— republished with incremented retry count. Reason: {reason}"
+            )
+        else:
+            logger.warning(
+                f"Message {correlation_id} exceeded max retries ({self.config.max_retries}) "
+                f"— sending to DLQ. Reason: {reason}"
+            )
+        await message.reject(requeue=False)
+
     async def callback(
         self,
         message: aio_pika.abc.AbstractIncomingMessage,
@@ -268,41 +307,63 @@ class BertrendWorker:
 
             logger.info(f"Request endpoint: {request_data.get('endpoint')}")
 
-            # Process request
-            response_data = await self.process_request(request_data)
+            # Process request with a timeout to avoid blocking the queue indefinitely
+            response_data = await asyncio.wait_for(
+                self.process_request(request_data),
+                timeout=self.config.job_timeout,
+            )
 
             # Add metadata
             response_data["correlation_id"] = correlation_id
             response_data["request_data"] = request_data
 
-            # Publish response/error if reply_to is specified
-            if message.reply_to:
-                if response_data.get("status") == "error":
-                    await self.queue_manager.publish_error(
-                        error_data=response_data, correlation_id=correlation_id
-                    )
-                else:
-                    await self.queue_manager.publish_response(
-                        response_data=response_data, correlation_id=correlation_id
-                    )
-
-            # Acknowledge message
-            await message.ack()
-            logger.info(
-                f"Completed request: {correlation_id} - Status: {response_data.get('status')}"
-            )
-
         except json.JSONDecodeError as e:
             logger.error(f"Invalid message format: {str(e)}")
             # Reject and don't requeue invalid messages
             await message.reject(requeue=False)
+            return
+
+        except asyncio.TimeoutError:
+            await self._nack_or_discard(
+                message,
+                reason=f"job timed out after {self.config.job_timeout}s",
+                correlation_id=correlation_id,
+            )
+            return
 
         except Exception as e:
             logger.error(f"Error processing message: {str(e)}")
             logger.error(traceback.format_exc())
+            await self._nack_or_discard(
+                message,
+                reason=str(e),
+                correlation_id=correlation_id,
+            )
+            return
 
-            # Requeue for retry (will go to DLQ after max retries)
-            await message.nack(requeue=True)
+        # Acknowledge message before publishing the response, so that the message
+        # is always acked regardless of any subsequent publish failure
+        await message.ack()
+        logger.info(
+            f"Completed request: {correlation_id} - Status: {response_data.get('status')}"
+        )
+
+        # Publish response/error if reply_to is specified
+        if message.reply_to:
+            try:
+                if response_data.get("status") == "error":
+                    await self.queue_manager.publish_error(
+                        error_data=response_data, correlation_id=correlation_id
+                    )
+                else:
+                    await self.queue_manager.publish_response(
+                        response_data=response_data, correlation_id=correlation_id
+                    )
+            except Exception as e:
+                logger.error(
+                    f"Failed to publish response for {correlation_id}: {str(e)}"
+                )
+                logger.error(traceback.format_exc())
 
     async def start(self):
         """Start the worker"""

{bertrend-0.4.8 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_management/queue_manager.py

@@ -159,6 +159,55 @@ class QueueManager:
 
         await queue.consume(callback, no_ack=False)
 
+    async def republish_with_retry(
+        self,
+        original_message: aio_pika.abc.AbstractIncomingMessage,
+        retry_count: int,
+    ) -> bool:
+        """Republish a failed message with an incremented x-retry-count header.
+
+        RabbitMQ does not modify headers on requeue, so we must republish
+        manually with the updated counter instead of using nack(requeue=True).
+
+        Returns True if the message was republished, False if the retry limit
+        has already been reached and the message was not republished.
+        """
+        new_retry_count = retry_count + 1
+        if new_retry_count > self.config.max_retries:
+            logger.warning(
+                f"Message {original_message.correlation_id} has reached the maximum "
+                f"retry limit ({self.config.max_retries}) — not republishing."
+            )
+            return False
+
+        if not self.channel or self.channel.is_closed:
+            await self.connect()
+
+        # Carry over all existing headers and bump the retry counter
+        headers = dict(original_message.headers or {})
+        headers["x-retry-count"] = new_retry_count
+
+        message = aio_pika.Message(
+            body=original_message.body,
+            headers=headers,
+            delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
+            priority=original_message.priority,
+            correlation_id=original_message.correlation_id,
+            content_type=original_message.content_type,
+            reply_to=original_message.reply_to,
+        )
+
+        await self.channel.default_exchange.publish(
+            message,
+            routing_key=self.config.request_queue,
+        )
+
+        logger.info(
+            f"Republished message {original_message.correlation_id} "
+            f"with x-retry-count={new_retry_count}"
+        )
+        return True
+
     async def publish_response(self, response_data: dict, correlation_id: str):
         """Publish response to response queue_management"""
         if not self.channel or self.channel.is_closed:

{bertrend-0.4.8 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_management/rabbitmq_config.py

@@ -36,5 +36,8 @@ class RabbitMQConfig:
     prefetch_count: int = int(os.getenv("RABBITMQ_PREFETCH_COUNT", 1))
 
     # Retry configuration
-    max_retries: int = int(os.getenv("RABBITMQ_MAX_RETRIES", 3))
+    max_retries: int = int(os.getenv("RABBITMQ_MAX_RETRIES", 2))
     retry_delay: int = int(os.getenv("RABBITMQ_RETRY_DELAY", 5000))  # milliseconds
+
+    # Job timeout: max seconds a single job may run before being nacked (default: 1 hour)
+    job_timeout: int = int(os.getenv("RABBITMQ_JOB_TIMEOUT", 3600))

{bertrend-0.4.8 → bertrend-0.4.9}/bertrend/bertrend_apps/services/queue_monitor/app.py

@@ -60,6 +60,18 @@ class RabbitMQAdminClient:
         r.raise_for_status()
         return r.json()
 
+    def get_consumers(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/api/consumers/{self._vhost_path()}"
+        r = self._session.get(url, timeout=5)
+        r.raise_for_status()
+        return r.json()
+
+    def get_channels(self) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/api/channels"
+        r = self._session.get(url, timeout=5)
+        r.raise_for_status()
+        return r.json()
+
     def peek_messages(self, name: str, count: int = 10) -> List[Dict[str, Any]]:
         url = f"{self.base_url}/api/queues/{self._vhost_path()}/{quote(name, safe='')}/get"
         payload = {
@@ -262,7 +274,14 @@ client = RabbitMQAdminClient(client_cfg, management_url=mgmt_url)
 
 # --- Data Fetching ---
 queue_data = {}
-global_stats = {"total": 0, "ready": 0, "unacked": 0, "consumers": 0}
+global_stats = {
+    "total": 0,
+    "requests": 0,
+    "responses": 0,
+    "failed": 0,
+    "unacked": 0,
+    "consumers": 0,
+}
 error_msg = None
 
 try:
@@ -272,9 +291,14 @@ try:
 
         # Aggregation
         global_stats["total"] += info.get("messages", 0)
-        global_stats["ready"] += info.get("messages_ready", 0)
         global_stats["unacked"] += info.get("messages_unacknowledged", 0)
         global_stats["consumers"] += info.get("consumers", 0)
+        if "requests" in q:
+            global_stats["requests"] += info.get("messages", 0)
+        elif "responses" in q or "response" in q:
+            global_stats["responses"] += info.get("messages", 0)
+        elif "failed" in q:
+            global_stats["failed"] += info.get("messages", 0)
 
 except Exception as e:
     error_msg = str(e)
@@ -288,18 +312,33 @@ if error_msg:
 
 # 1. Global Overview Row
 st.markdown(f"### {DASHBOARD_ICON} System Overview")
-kpi1, kpi2, kpi3, kpi4 = st.columns(4)
+kpi1, kpi2, kpi3, kpi4, kpi5, kpi6 = st.columns(6)
 kpi1.metric(
     "Total Backlog", global_stats["total"], help="Total messages in all selected queues"
 )
-kpi2.metric("Ready to Process", global_stats["ready"], delta_color="normal")
+kpi2.metric(
+    "Requests",
+    global_stats["requests"],
+    help="Messages in bertrend_requests queue",
+)
 kpi3.metric(
+    "Responses",
+    global_stats["responses"],
+    help="Messages in bertrend_responses queue",
+)
+kpi4.metric(
+    "Failed",
+    global_stats["failed"],
+    delta_color="inverse",
+    help="Messages in bertrend_failed queue",
+)
+kpi5.metric(
     "Unacknowledged",
     global_stats["unacked"],
     delta_color="inverse",
     help="Messages currently being processed but not acked",
 )
-kpi4.metric("Active Consumers", global_stats["consumers"])
+kpi6.metric("Active Consumers", global_stats["consumers"])
 
 st.divider()
 
@@ -327,6 +366,108 @@ else:
 
             st.divider()
 
+            # B. Unacknowledged Messages Details
+            unacked_count = info.get("messages_unacknowledged", 0)
+            with st.expander(
+                f"Unacknowledged Messages ({unacked_count})",
+                expanded=unacked_count > 0,
+            ):
+                if unacked_count == 0:
+                    st.info("No unacknowledged messages in this queue.", icon=INFO_ICON)
+                else:
+                    try:
+                        all_consumers = client.get_consumers()
+                        all_channels = client.get_channels()
+
+                        # Filter consumers for this queue
+                        queue_consumers = [
+                            c
+                            for c in all_consumers
+                            if c.get("queue", {}).get("name") == qname
+                        ]
+
+                        # Build a map of channel_name -> channel details
+                        channel_map = {ch.get("name"): ch for ch in all_channels}
+
+                        if not queue_consumers:
+                            st.warning(
+                                "Unacknowledged messages exist but no active consumers found. "
+                                "Messages may be held by a disconnected consumer."
+                            )
+                        else:
+                            st.caption(
+                                f"{len(queue_consumers)} consumer(s) holding unacknowledged messages:"
+                            )
+                            for cons in queue_consumers:
+                                ch_name = cons.get("channel_details", {}).get(
+                                    "name", ""
+                                )
+                                ch = channel_map.get(ch_name, {})
+                                ch_unacked = ch.get("messages_unacknowledged", 0)
+                                consumer_tag = cons.get("consumer_tag", "unknown")
+                                peer_host = cons.get("channel_details", {}).get(
+                                    "peer_host", "?"
+                                )
+                                peer_port = cons.get("channel_details", {}).get(
+                                    "peer_port", ""
+                                )
+                                prefetch = cons.get("prefetch_count", 0)
+                                ack_required = cons.get("ack_required", True)
+
+                                with st.container(border=True):
+                                    cc1, cc2, cc3, cc4 = st.columns(4)
+                                    cc1.metric("Unacked (channel)", ch_unacked)
+                                    cc2.markdown(f"**Consumer Tag**\n`{consumer_tag}`")
+                                    cc3.markdown(
+                                        f"**Client**\n`{peer_host}:{peer_port}`"
+                                    )
+                                    cc4.markdown(
+                                        f"**Prefetch / Ack required**\n`{prefetch}` / `{ack_required}`"
+                                    )
+
+                                    # Channel-level stats
+                                    ch_stats = ch.get("message_stats", {})
+                                    ack_rate = (
+                                        ch_stats.get("ack_details", {}).get("rate", 0.0)
+                                        if ch_stats
+                                        else 0.0
+                                    )
+                                    deliver_rate = (
+                                        ch_stats.get("deliver_details", {}).get(
+                                            "rate", 0.0
+                                        )
+                                        if ch_stats
+                                        else 0.0
+                                    )
+                                    total_acked = (
+                                        ch_stats.get("ack", 0) if ch_stats else 0
+                                    )
+                                    total_delivered = (
+                                        ch_stats.get("deliver", 0) if ch_stats else 0
+                                    )
+
+                                    sc1, sc2, sc3, sc4 = st.columns(4)
+                                    sc1.metric("Ack Rate", f"{ack_rate:.2f}/s")
+                                    sc2.metric("Deliver Rate", f"{deliver_rate:.2f}/s")
+                                    sc3.metric("Total Acked", total_acked)
+                                    sc4.metric("Total Delivered", total_delivered)
+
+                                    # Connection info
+                                    conn = ch.get("connection_details", {})
+                                    if conn:
+                                        st.caption(
+                                            f"Connection: `{conn.get('name', ch_name)}` "
+                                            f"— state: `{ch.get('state', '?')}`"
+                                        )
+
+                    except Exception as e:
+                        st.error(
+                            f"Failed to fetch consumer/channel details: {e}",
+                            icon=ERROR_ICON,
+                        )
+
+            st.divider()
+
             # B. Message Peeking Interface
             c_head, c_peek_n, c_sort = st.columns([3, 1, 1])
             c_head.subheader("Message Browser")
@@ -546,6 +687,7 @@ else:
                     obj = d["obj"]
 
                     props = m.get("properties", {})
+                    headers = props.get("headers") or {}
                     corr_id = props.get("correlation_id", "No-ID")
                     msg_size = (
                         m.get("payload_bytes", 0)
@@ -562,6 +704,9 @@ else:
                         info_tags.append(f"User: {d['user']}")
                     if d["model_id"]:
                         info_tags.append(f"Model: {d['model_id']}")
+                    retry_count = headers.get("x-retry-count")
+                    if retry_count is not None:
+                        info_tags.append(f"Retry: {retry_count}")
 
                     info_str = f" | {' | '.join(info_tags)}" if info_tags else ""
                     label = f"#{idx + 1} | {fmt} | CID: {corr_id}{info_str} | {msg_size} bytes"
@@ -617,6 +762,12 @@ else:
                                             "Key": "App ID",
                                             "Val": str(props.get("app_id") or ""),
                                         },
+                                        {
+                                            "Key": "x-retry-count",
+                                            "Val": str(headers.get("x-retry-count"))
+                                            if headers.get("x-retry-count") is not None
+                                            else "",
+                                        },
                                     ]
                                 ).dropna()
                                 st.dataframe(meta_df, hide_index=True, width="stretch")

bertrend-0.4.9/bertrend/tests/services/article_scoring/test_article_scoring_server.py

@@ -0,0 +1,84 @@
+#  Copyright (c) 2024-2026, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from unittest.mock import patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+from bertrend.article_scoring.article_scoring import ArticleScore, CriteriaScores
+from bertrend.llm_utils.agent_utils import ProcessingResult
+from bertrend.services.article_scoring_server.scoring_service import app
+
+
+@pytest.fixture
+def client():
+    return TestClient(app)
+
+
+def _make_article_score(
+    depth: int = 4, originality: int = 3, relevance_to_rte: int = 5
+):
+    return ArticleScore(
+        scores=CriteriaScores(
+            depth_of_reporting=depth / 5,
+            originality_and_exclusivity=originality / 5,
+            source_quality_and_transparency=0.8,
+            accuracy_and_fact_checking_rigor=0.8,
+            clarity_and_accessibility=0.8,
+            balance_and_fairness=0.8,
+            narrative_and_engagement=0.8,
+            timeliness_and_relevance=0.8,
+            ethical_considerations_and_sensitivity=0.8,
+            rte_relevance_and_strategic_impact=relevance_to_rte / 5,
+        )
+    )
+
+
+def test_health(client):
+    response = client.get("/health")
+    assert response.status_code == 200
+    assert response.json() == {"status": "ok"}
+
+
+def test_score_articles_success(client):
+    mock_results = [
+        ProcessingResult(input_index=0, output=_make_article_score()),
+        ProcessingResult(
+            input_index=1, output=_make_article_score(depth=2), error="timeout"
+        ),
+    ]
+
+    with patch(
+        "bertrend.services.article_scoring_server.routers.scoring.score_articles",
+        return_value=mock_results,
+    ) as mock_score:
+        response = client.post("/score", json={"articles": ["article 1", "article 2"]})
+
+    assert response.status_code == 200
+    mock_score.assert_called_once_with(["article 1", "article 2"])
+
+    data = response.json()
+    assert len(data["results"]) == 2
+    assert data["results"][0]["article_index"] == 0
+    assert data["results"][0]["quality_metrics"] is not None
+    assert data["results"][0]["overall_quality"] == "POOR"
+    assert data["results"][0]["error"] is None
+
+    assert data["results"][1]["article_index"] == 1
+    assert data["results"][1]["quality_metrics"] is not None
+    assert data["results"][1]["overall_quality"] is None
+    assert data["results"][1]["error"] == "timeout"
+
+
+def test_score_articles_backend_exception(client):
+    with patch(
+        "bertrend.services.article_scoring_server.routers.scoring.score_articles",
+        side_effect=RuntimeError("provider down"),
+    ):
+        response = client.post("/score", json={"articles": ["article 1"]})
+
+    assert response.status_code == 500
+    assert "provider down" in response.json()["detail"]