locust 2.28.1.dev54__py3-none-any.whl → 2.28.1.dev66__py3-none-any.whl

locust/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.28.1.dev54'
-__version_tuple__ = version_tuple = (2, 28, 1, 'dev54')
+__version__ = version = '2.28.1.dev66'
+__version_tuple__ = version_tuple = (2, 28, 1, 'dev66')

locust/env.py

@@ -99,6 +99,8 @@ class Environment:
         """List of the available Tasks per User Classes to pick from in the Task Picker"""
         self.dispatcher_class = dispatcher_class
         """A user dispatcher class that decides how users are spawned, default :class:`UsersDispatcher <locust.dispatch.UsersDispatcher>`"""
+        self.worker_logs: dict[str, list[str]] = {}
+        """Captured logs from all connected workers"""
 
         self._remove_user_classes_with_weight_zero()
         self._validate_user_class_name_uniqueness()
@@ -209,6 +211,10 @@ class Environment:
             if key == "tasks":
                 user_class.tasks = [task for task in user_tasks if task.__name__ in value]
 
+    def update_worker_logs(self, worker_log_report):
+        if worker_log_report.get("worker_id", None):
+            self.worker_logs[worker_log_report.get("worker_id")] = worker_log_report.get("logs", [])
+
     def _filter_tasks_by_tags(self) -> None:
         """
         Filter the tasks on all the user_classes recursively, according to the tags and

locust/log.py

@@ -2,6 +2,7 @@ import logging
 import logging.config
 import re
 import socket
+from collections import deque
 
 HOSTNAME = re.sub(r"\..*", "", socket.gethostname())
 
@@ -13,7 +14,7 @@ unhandled_greenlet_exception = False
 class LogReader(logging.Handler):
     def __init__(self):
         super().__init__()
-        self.logs = []
+        self.logs = deque(maxlen=500)
 
     def emit(self, record):
         self.logs.append(self.format(record))
@@ -75,6 +76,15 @@ def setup_logging(loglevel, logfile=None):
     logging.config.dictConfig(LOGGING_CONFIG)
 
 
+def get_logs():
+    log_reader_handler = [handler for handler in logging.getLogger("root").handlers if handler.name == "log_reader"]
+
+    if log_reader_handler:
+        return list(log_reader_handler[0].logs)
+
+    return []
+
+
 def greenlet_exception_logger(logger, level=logging.CRITICAL):
     """
     Return a function that can be used as argument to Greenlet.link_exception() that will log the

locust/runners.py

@@ -39,7 +39,7 @@ from gevent.pool import Group
 from . import argument_parser
 from .dispatch import UsersDispatcher
 from .exception import RPCError, RPCReceiveError, RPCSendError
-from .log import greenlet_exception_logger
+from .log import get_logs, greenlet_exception_logger
 from .rpc import (
     Message,
     rpc,
@@ -66,6 +66,7 @@ STATE_INIT, STATE_SPAWNING, STATE_RUNNING, STATE_CLEANUP, STATE_STOPPING, STATE_
     "missing",
 ]
 WORKER_REPORT_INTERVAL = 3.0
+WORKER_LOG_REPORT_INTERVAL = 10
 CPU_MONITOR_INTERVAL = 5.0
 CPU_WARNING_THRESHOLD = 90
 HEARTBEAT_INTERVAL = 1
@@ -1116,6 +1117,8 @@ class MasterRunner(DistributedRunner):
                 # a worker finished spawning (this happens multiple times during rampup)
                 self.clients[msg.node_id].state = STATE_RUNNING
                 self.clients[msg.node_id].user_classes_count = msg.data["user_classes_count"]
+            elif msg.type == "logs":
+                self.environment.update_worker_logs(msg.data)
             elif msg.type == "quit":
                 if msg.node_id in self.clients:
                     client = self.clients[msg.node_id]
@@ -1212,6 +1215,7 @@ class WorkerRunner(DistributedRunner):
         self.client_id = socket.gethostname() + "_" + uuid4().hex
         self.master_host = master_host
         self.master_port = master_port
+        self.logs: list[str] = []
         self.worker_cpu_warning_emitted = False
         self._users_dispatcher: UsersDispatcher | None = None
         self.client = rpc.Client(master_host, master_port, self.client_id)
@@ -1220,6 +1224,7 @@ class WorkerRunner(DistributedRunner):
         self.greenlet.spawn(self.heartbeat).link_exception(greenlet_exception_handler)
         self.greenlet.spawn(self.heartbeat_timeout_checker).link_exception(greenlet_exception_handler)
         self.greenlet.spawn(self.stats_reporter).link_exception(greenlet_exception_handler)
+        self.greenlet.spawn(self.logs_reporter).link_exception(greenlet_exception_handler)
 
         # register listener that adds the current number of spawned users to the report that is sent to the master node
         def on_report_to_master(client_id: str, data: dict[str, Any]):
@@ -1417,6 +1422,25 @@ class WorkerRunner(DistributedRunner):
                 logger.error(f"Temporary connection lost to master server: {e}, will retry later.")
             gevent.sleep(WORKER_REPORT_INTERVAL)
 
+    def logs_reporter(self) -> None:
+        if WORKER_LOG_REPORT_INTERVAL < 0:
+            return
+
+        while True:
+            current_logs = get_logs()
+
+            if (len(current_logs) - len(self.logs)) > 10:
+                logger.warning(
+                    "The worker attempted to send more than 10 log lines in one interval. Further log sending was disabled for this worker."
+                )
+                self._send_logs(get_logs())
+                break
+            if len(current_logs) > len(self.logs):
+                self._send_logs(current_logs)
+
+            self.logs = current_logs
+            gevent.sleep(WORKER_LOG_REPORT_INTERVAL)
+
     def send_message(self, msg_type: str, data: dict[str, Any] | None = None, client_id: str | None = None) -> None:
         """
         Sends a message to master node
@@ -1433,6 +1457,9 @@ class WorkerRunner(DistributedRunner):
         self.environment.events.report_to_master.fire(client_id=self.client_id, data=data)
         self.client.send(Message("stats", data, self.client_id))
 
+    def _send_logs(self, current_logs) -> None:
+        self.send_message("logs", {"worker_id": self.client_id, "logs": current_logs})
+
     def connect_to_master(self):
         self.retry += 1
         self.client.send(Message("client_ready", __version__, self.client_id))

locust/test/test_runners.py

@@ -11,6 +11,7 @@ from locust.argument_parser import parse_options
 from locust.dispatch import UsersDispatcher
 from locust.env import Environment
 from locust.exception import RPCError, RPCReceiveError, StopUser
+from locust.log import LogReader
 from locust.main import create_environment
 from locust.rpc import Message
 from locust.runners import (
@@ -32,6 +33,7 @@ from locust.user import (
 )
 
 import json
+import logging
 import random
 import time
 import unittest
@@ -80,11 +82,12 @@ def mocked_rpc(raise_on_close=True):
             self.outbox.append(message)
 
         def send_to_client(self, message):
-            self.outbox.append((message.node_id, message))
+            print(message)
+            self.outbox.append(message)
 
         @classmethod
         def get_messages(cls, message_type=None) -> list:
-            return [message[1] for message in cls.outbox if message_type is None or message[1].type == message_type]
+            return [message for message in cls.outbox if message_type is None or message.type == message_type]
 
         def recv_from_client(self):
             results = self.queue.get()
@@ -2601,9 +2604,9 @@ class TestMasterRunner(LocustRunnerTestCase):
             server.mocked_send(Message("client_ready", __version__, "zeh_fake_client2"))
             self.assertEqual(2, len(master.clients))
             sleep(0.1)  # give time for messages to be sent to clients
-            spawn_messages = [message for message in server.outbox if message[1].type == "spawn"]
-            self.assertEqual({"TestUser": 50}, spawn_messages[-1][1].data["user_classes_count"])
-            self.assertEqual({"TestUser": 50}, spawn_messages[-2][1].data["user_classes_count"])
+            spawn_messages = server.get_messages("spawn")
+            self.assertEqual({"TestUser": 50}, spawn_messages[-1].data["user_classes_count"])
+            self.assertEqual({"TestUser": 50}, spawn_messages[-2].data["user_classes_count"])
 
     def test_sends_spawn_data_to_ready_running_spawning_workers(self):
         """Sends spawn job to running, ready, or spawning workers"""
@@ -2648,7 +2651,7 @@ class TestMasterRunner(LocustRunnerTestCase):
                 server.mocked_send(Message("client_ready", __version__, "fake_client%i" % i))
 
             master.start(7, 7)
-            self.assertEqual(15, len(server.outbox))
+            self.assertEqual(15, len(server.get_messages()))
             self.assertEqual(1, run_count[0])
 
             # change number of users and check that test_start isn't fired again
@@ -2687,7 +2690,7 @@ class TestMasterRunner(LocustRunnerTestCase):
                 server.mocked_send(Message("client_ready", __version__, "fake_client%i" % i))
 
             master.start(7, 7)
-            self.assertEqual(15, len(server.outbox))
+            self.assertEqual(15, len(server.get_messages()))
             master.stop()
             self.assertTrue(self.runner_stopping)
             self.assertTrue(self.runner_stopped)
@@ -2726,7 +2729,7 @@ class TestMasterRunner(LocustRunnerTestCase):
                 server.mocked_send(Message("client_ready", __version__, "fake_client%i" % i))
 
             master.start(7, 7)
-            self.assertEqual(15, len(server.outbox))
+            self.assertEqual(15, len(server.get_messages()))
             master.quit()
             self.assertTrue(self.runner_stopping)
             self.assertTrue(self.runner_stopped)
@@ -2819,9 +2822,8 @@ class TestMasterRunner(LocustRunnerTestCase):
             self.assertEqual(USERS_COUNT * 3, len(server.outbox))
 
             indexes = []
-            for _, msg in server.outbox:
-                if msg.type == "ack":
-                    indexes.append(msg.data["index"])
+            for msg in server.get_messages("ack"):
+                indexes.append(msg.data["index"])
             self.assertEqual(USERS_COUNT, len(indexes), "Total number of locusts/workers is not 5")
 
             indexes.sort()
@@ -3110,8 +3112,10 @@ class TestMasterRunner(LocustRunnerTestCase):
                 master.clients[i] = WorkerNode(str(i))
             master.send_message("test_custom_msg", {"test_data": 123})
 
-            self.assertEqual(5, len(server.outbox))
-            for _, msg in server.outbox:
+            messages = server.get_messages()
+
+            self.assertEqual(5, len(messages))
+            for msg in messages:
                 self.assertEqual("test_custom_msg", msg.type)
                 self.assertEqual(123, msg.data["test_data"])
 
@@ -3198,10 +3202,12 @@ class TestMasterRunner(LocustRunnerTestCase):
             master = self.get_runner()
             server.mocked_send(Message("client_ready", __version__, "dummy_client"))
 
+            messages = server.get_messages()
+
             self.assertEqual(1, len(master.clients))
-            self.assertEqual("ack", server.outbox[0][1].type)
-            self.assertEqual(1, len(server.outbox))
-            self.assertEqual(0, server.outbox[0][1].data["index"])
+            self.assertEqual("ack", messages[0].type)
+            self.assertEqual(1, len(messages))
+            self.assertEqual(0, messages[0].data["index"])
 
     def test_worker_sends_bad_message_to_master(self):
         """
@@ -3224,11 +3230,12 @@ class TestMasterRunner(LocustRunnerTestCase):
             master.start(10, 10)
             sleep(0.1)
             server.mocked_send(Message("stats", BAD_MESSAGE, "zeh_fake_client1"))
-            self.assertEqual(5, len(server.outbox))
+            messages = server.get_messages()
+            self.assertEqual(5, len(messages))
 
             # Expected message order in outbox: ack, spawn, reconnect, ack
             self.assertEqual(
-                "reconnect", server.outbox[3][1].type, "Master didn't send worker reconnect message when expected."
+                "reconnect", messages[3].type, "Master didn't send worker reconnect message when expected."
             )
 
     def test_worker_sends_unrecognized_message_to_master(self):
@@ -3252,7 +3259,7 @@ class TestMasterRunner(LocustRunnerTestCase):
             master.start(10, 10)
             sleep(0.1)
             server.mocked_send(Message("stats", UNRECOGNIZED_MESSAGE, "zeh_fake_client1"))
-            self.assertEqual(3, len(server.outbox))
+            self.assertEqual(3, len(server.get_messages()))
 
     def test_unknown_host_sends_message_to_master(self):
         """
@@ -3275,7 +3282,7 @@ class TestMasterRunner(LocustRunnerTestCase):
             master.start(10, 10)
             sleep(0.1)
             server.mocked_send(Message("stats", UNRECOGNIZED_HOST_MESSAGE, "unknown_host"))
-            self.assertEqual(3, len(server.outbox))
+            self.assertEqual(3, len(server.get_messages()))
 
 
 class TestWorkerRunner(LocustTestCase):
@@ -3308,8 +3315,9 @@ class TestWorkerRunner(LocustTestCase):
 
         with mock.patch("locust.rpc.rpc.Client", mocked_rpc()) as client:
             worker = self.get_runner(environment=Environment(), user_classes=[MyTestUser], client=client)
-            self.assertEqual(1, len(client.outbox))
-            self.assertEqual("client_ready", client.outbox[0].type)
+            messages = client.get_messages()
+            self.assertEqual(1, len(messages))
+            self.assertEqual("client_ready", messages[0].type)
             client.mocked_send(
                 Message(
                     "spawn",
@@ -3323,8 +3331,9 @@ class TestWorkerRunner(LocustTestCase):
                     "dummy_client_id",
                 )
             )
+            self.assertTrue(client.get_messages("spawning"))
             # wait for worker to spawn locusts
-            self.assertIn("spawning", [m.type for m in client.outbox])
+            # self.assertIn("spawning", [m.type for m in messages])
             worker.spawning_greenlet.join()
             self.assertEqual(1, len(worker.user_greenlets))
             # check that locust has started running
@@ -3348,8 +3357,9 @@ class TestWorkerRunner(LocustTestCase):
 
         with mock.patch("locust.rpc.rpc.Client", mocked_rpc()) as client:
             worker = self.get_runner(environment=Environment(), user_classes=[MyTestUser], client=client)
-            self.assertEqual(1, len(client.outbox))
-            self.assertEqual("client_ready", client.outbox[0].type)
+            messages = client.get_messages()
+            self.assertEqual(1, len(messages))
+            self.assertEqual("client_ready", messages[0].type)
             client.mocked_send(
                 Message(
                     "spawn",
@@ -3365,7 +3375,7 @@ class TestWorkerRunner(LocustTestCase):
             )
             # print("outbox:", client.outbox)
             # wait for worker to spawn locusts
-            self.assertIn("spawning", [m.type for m in client.outbox])
+            self.assertTrue(client.get_messages("spawning"))
             worker.spawning_greenlet.join()
             self.assertEqual(1, len(worker.user_greenlets))
             # check that locust has started running
@@ -3509,14 +3519,14 @@ class TestWorkerRunner(LocustTestCase):
 
             sleep(2)
 
-            message = next((m for m in reversed(client.outbox) if m.type == "stats"), None)
+            message = client.get_messages("stats")[-1]
             self.assertIsNotNone(message)
             self.assertIn("user_count", message.data)
             self.assertIn("user_classes_count", message.data)
             self.assertEqual(message.data["user_count"], 10)
             self.assertEqual(message.data["user_classes_count"]["MyUser"], 10)
 
-            message = next((m for m in client.outbox if m.type == "spawning_complete"), None)
+            message = client.get_messages("spawning_complete")[0]
             self.assertIsNotNone(message)
             self.assertIn("user_count", message.data)
             self.assertIn("user_classes_count", message.data)
@@ -3541,11 +3551,11 @@ class TestWorkerRunner(LocustTestCase):
             worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)
 
             t0 = time.perf_counter()
-            while len([m for m in client.outbox if m.type == "heartbeat"]) == 0:
+            while len(client.get_messages("heartbeat")) == 0:
                 self.assertLessEqual(time.perf_counter() - t0, 3)
                 sleep(0.1)
 
-            message = next(m for m in reversed(client.outbox) if m.type == "heartbeat")
+            message = client.get_messages("heartbeat")[-1]
             self.assertEqual(len(message.data), 3)
             self.assertIn("state", message.data)
             self.assertIn("current_cpu_usage", message.data)
@@ -3718,8 +3728,9 @@ class TestWorkerRunner(LocustTestCase):
             worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)
             client.outbox.clear()
             worker.send_message("test_custom_msg", {"test_data": 123})
-            self.assertEqual("test_custom_msg", client.outbox[0].type)
-            self.assertEqual(123, client.outbox[0].data["test_data"])
+            messages = client.get_messages()
+            self.assertEqual("test_custom_msg", messages[0].type)
+            self.assertEqual(123, messages[0].data["test_data"])
             worker.quit()
 
     def test_custom_message_receive(self):
@@ -3790,8 +3801,9 @@ class TestWorkerRunner(LocustTestCase):
                 run_count[0] += 1
 
             worker = self.get_runner(environment=environment, user_classes=[MyTestUser], client=client)
-            self.assertEqual(1, len(client.outbox))
-            self.assertEqual("client_ready", client.outbox[0].type)
+            messages = client.get_messages()
+            self.assertEqual(1, len(messages))
+            self.assertEqual("client_ready", messages[0].type)
             client.mocked_send(
                 Message(
                     "spawn",
@@ -3808,7 +3820,7 @@ class TestWorkerRunner(LocustTestCase):
                 )
             )
             # wait for worker to spawn locusts
-            self.assertIn("spawning", [m.type for m in client.outbox])
+            self.assertTrue(client.get_messages("spawning"))
             worker.spawning_greenlet.join()
             self.assertEqual(1, len(worker.user_greenlets))
             self.assertEqual(1, run_count[0])
@@ -3876,8 +3888,9 @@ class TestWorkerRunner(LocustTestCase):
                 run_count[0] += 1
 
             worker = self.get_runner(environment=environment, user_classes=[MyTestUser], client=client)
-            self.assertEqual(1, len(client.outbox))
-            self.assertEqual("client_ready", client.outbox[0].type)
+            messages = client.get_messages()
+            self.assertEqual(1, len(messages))
+            self.assertEqual("client_ready", messages[0].type)
             client.mocked_send(
                 Message(
                     "spawn",
@@ -3895,7 +3908,7 @@ class TestWorkerRunner(LocustTestCase):
             )
 
             # wait for worker to spawn locusts
-            self.assertIn("spawning", [m.type for m in client.outbox])
+            self.assertTrue(client.get_messages("spawning"))
             worker.spawning_greenlet.join()
             self.assertEqual(1, len(worker.user_greenlets))
 
@@ -3942,9 +3955,10 @@ class TestWorkerRunner(LocustTestCase):
         with mock.patch("locust.runners.CONNECT_TIMEOUT", new=1):
             with mock.patch("locust.rpc.rpc.Client", mocked_rpc()) as client:
                 worker = self.get_runner(environment=Environment(), user_classes=[MyTestUser], client=client)
+                messages = client.get_messages()
 
-                self.assertEqual("client_ready", client.outbox[0].type)
-                self.assertEqual(1, len(client.outbox))
+                self.assertEqual("client_ready", messages[0].type)
+                self.assertEqual(1, len(messages))
                 self.assertTrue(worker.connected)
 
     def test_worker_connect_failure(self):
@@ -3962,6 +3976,71 @@ class TestWorkerRunner(LocustTestCase):
                         )
                     self.assertEqual(2, len(client.outbox))
 
+    def test_send_logs(self):
+        class MyUser(User):
+            wait_time = constant(1)
+
+            @task
+            def my_task(self):
+                pass
+
+        with mock.patch("locust.rpc.rpc.Client", mocked_rpc()) as client:
+            short_time = 0.05
+
+            log_handler = LogReader()
+            log_handler.name = "log_reader"
+            log_handler.setLevel(logging.INFO)
+            logger = logging.getLogger("root")
+            logger.addHandler(log_handler)
+            log_line = "some log info"
+            logger.info(log_line)
+
+            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)
+
+            gevent.sleep(short_time)
+
+            messages = client.get_messages()
+
+            self.assertEqual("logs", messages[3].type)
+            self.assertEqual(log_line, messages[3].data.get("logs", [])[0])
+            self.assertEqual(worker.client_id, messages[3].data.get("worker_id"))
+            worker.quit()
+
+    def test_quit_worker_logs(self):
+        class MyUser(User):
+            wait_time = constant(1)
+
+            @task
+            def my_task(self):
+                pass
+
+        with mock.patch("locust.rpc.rpc.Client", mocked_rpc()) as client:
+            short_time = 0.05
+
+            log_handler = LogReader()
+            log_handler.name = "log_reader"
+            log_handler.setLevel(logging.INFO)
+            logger = logging.getLogger("root")
+            logger.addHandler(log_handler)
+            log_line = "spamming log"
+
+            for _ in range(11):
+                logger.info(log_line)
+
+            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)
+
+            gevent.sleep(short_time)
+
+            message = client.get_messages("logs")[0]
+
+            self.assertEqual(
+                "The worker attempted to send more than 10 log lines in one interval. Further log sending was disabled for this worker.",
+                message.data.get("logs", [])[-1],
+            )
+            self.assertEqual(worker.client_id, message.data.get("worker_id"))
+            worker.quit()
+            logger.removeHandler(log_handler)
+
 
 class TestMessageSerializing(unittest.TestCase):
     def test_message_serialize(self):

locust/test/test_web.py

@@ -1020,7 +1020,25 @@ class TestWebUI(LocustTestCase, _HeaderCheckMixin):
 
         response = requests.get("http://127.0.0.1:%i/logs" % self.web_port)
 
-        self.assertIn(log_line, response.json().get("logs"))
+        self.assertIn(log_line, response.json().get("master"))
+
+    def test_worker_logs(self):
+        log_handler = LogReader()
+        log_handler.name = "log_reader"
+        log_handler.setLevel(logging.INFO)
+        logger = logging.getLogger("root")
+        logger.addHandler(log_handler)
+        log_line = "some log info"
+        logger.info(log_line)
+
+        worker_id = "123"
+        worker_log_line = "worker log"
+        self.environment.update_worker_logs({"worker_id": worker_id, "logs": [worker_log_line]})
+
+        response = requests.get("http://127.0.0.1:%i/logs" % self.web_port)
+
+        self.assertIn(log_line, response.json().get("master"))
+        self.assertIn(worker_log_line, response.json().get("workers").get(worker_id))
 
     def test_template_args(self):
         class MyUser(User):

locust/web.py

@@ -33,7 +33,7 @@ from . import __version__ as version
 from . import argument_parser
 from . import stats as stats_module
 from .html import BUILD_PATH, ROOT_PATH, STATIC_PATH, get_html_report
-from .log import greenlet_exception_logger
+from .log import get_logs, greenlet_exception_logger
 from .runners import STATE_MISSING, STATE_RUNNING, MasterRunner
 from .stats import StatsCSV, StatsCSVFileWriter, StatsErrorDict, sort_stats
 from .user.inspectuser import get_ratio
@@ -490,16 +490,7 @@ class WebUI:
         @app.route("/logs")
         @self.auth_required_if_enabled
         def logs():
-            log_reader_handler = [
-                handler for handler in logging.getLogger("root").handlers if handler.name == "log_reader"
-            ]
-
-            if log_reader_handler:
-                logs = log_reader_handler[0].logs
-            else:
-                logs = []
-
-            return jsonify({"logs": logs})
+            return jsonify({"master": get_logs(), "workers": self.environment.worker_logs})
 
         @app.route("/login")
         def login():