locust-plugins 4.2.0__tar.gz → 4.3.1__tar.gz

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/.pylintrc

@@ -11,7 +11,7 @@ attr-rgx=[a-z0-9_]{1,30}$
 ignored-argument-names=_.*|^ignored_|^unused_|^kwargs|^environment
 
 [MESSAGES CONTROL]
-disable=logging-not-lazy,logging-fstring-interpolation,missing-docstring,wrong-import-position,wrong-import-order,too-few-public-methods,invalid-name,protected-access,logging-format-interpolation,dangerous-default-value,global-statement,too-many-locals,too-many-arguments,too-many-instance-attributes,blacklisted-name,attribute-defined-outside-init,broad-except,bare-except,consider-using-with,too-many-branches,unspecified-encoding,arguments-differ,broad-exception-raised
+disable=logging-not-lazy,logging-fstring-interpolation,missing-docstring,wrong-import-position,wrong-import-order,too-few-public-methods,invalid-name,protected-access,logging-format-interpolation,dangerous-default-value,global-statement,too-many-locals,too-many-arguments,too-many-instance-attributes,blacklisted-name,attribute-defined-outside-init,broad-except,bare-except,consider-using-with,too-many-branches,unspecified-encoding,arguments-differ,broad-exception-raised,wildcard-import,keyword-arg-before-vararg
 
 [MASTER]
 ignore=locustio,examples

{locust-plugins-4.2.0/locust_plugins.egg-info → locust-plugins-4.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: locust-plugins
-Version: 4.2.0
+Version: 4.3.1
 Summary: Useful plugins/extensions for Locust
 Home-page: https://github.com/SvenskaSpel/locust-plugins
 Author: Lars Holmberg
@@ -16,20 +16,44 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Requires-Python: >=3.8, <4
+License-File: LICENSE
+Requires-Dist: locust>=2.20.0
+Requires-Dist: typing-extensions
 Provides-Extra: websocket
+Requires-Dist: websocket-client; extra == "websocket"
 Provides-Extra: playwright
+Requires-Dist: playwright>=1.34.0; extra == "playwright"
 Provides-Extra: dashboards
+Requires-Dist: psycogreen; extra == "dashboards"
+Requires-Dist: psycopg2-binary; extra == "dashboards"
 Provides-Extra: kafka
+Requires-Dist: confluent-kafka; extra == "kafka"
 Provides-Extra: mongo
+Requires-Dist: pymongo; extra == "mongo"
 Provides-Extra: mqtt
+Requires-Dist: paho-mqtt>=1.5.0; extra == "mqtt"
 Provides-Extra: appinsights
+Requires-Dist: opencensus-ext-azure; extra == "appinsights"
 Provides-Extra: resource
+Requires-Dist: lxml; extra == "resource"
 Provides-Extra: boto3
+Requires-Dist: boto3; extra == "boto3"
 Provides-Extra: all
+Requires-Dist: websocket-client; extra == "all"
+Requires-Dist: playwright>=1.34.0; extra == "all"
+Requires-Dist: psycogreen; extra == "all"
+Requires-Dist: psycopg2-binary; extra == "all"
+Requires-Dist: confluent-kafka; extra == "all"
+Requires-Dist: pymongo; extra == "all"
+Requires-Dist: paho-mqtt>=1.5.0; extra == "all"
+Requires-Dist: opencensus-ext-azure; extra == "all"
+Requires-Dist: lxml; extra == "all"
+Requires-Dist: boto3; extra == "all"
 Provides-Extra: webdriver
-License-File: LICENSE
+Requires-Dist: selenium>=4.0.0; extra == "webdriver"
 
 https://github.com/SvenskaSpel/locust-plugins

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/README.md

@@ -55,8 +55,8 @@ locust --help
 ## Wait time 
 - Custom wait time functions ([example](examples/constant_total_ips_ex.py), [source](locust_plugins/wait_time.py))
 
-## Debug 
-- Support for running a single User in the debugger (moved to [locust core](https://docs.locust.io/en/latest/running-in-debugger.html)!)
+## Distributing test data
+- Support for distributing test data from master to workers while maintaining test data order ([example](examples/distributor_ex.py), [source](locust_plugins/distributor.py))
 
 ## Transaction manager
 - Support for logging transactions (aggregating multiple requests or other actions) ([example](examples/transaction_example.py), [source](locust_plugins/transaction_manager.py))

locust-plugins-4.3.1/examples/distributor_ex.py

@@ -0,0 +1,41 @@
+# this example is a little more complex than it needs to be, but I wanted to highlight
+# that it is entirely possible to have multiple distributors at the same time
+
+from typing import Dict, List
+from locust_plugins.mongoreader import MongoLRUReader
+from locust_plugins.csvreader import CSVDictReader, CSVReader
+from locust_plugins.distributor import Distributor
+from locust import HttpUser, task, run_single_user, events
+from locust.runners import WorkerRunner
+
+distributors = {}
+
+
+@events.init.add_listener
+def on_locust_init(environment, **_kwargs):
+    ssn_reader = None
+    product_reader = None
+    if not isinstance(environment.runner, WorkerRunner):
+        product_reader = CSVReader("products.csv")
+        csv = True
+        if csv:
+            ssn_reader = CSVDictReader("ssn.tsv", delimiter="\t")
+        else:
+            ssn_reader = MongoLRUReader({"foo": "bar"}, "last_login")
+        product_reader = CSVReader("products.csv")
+    distributors["customers"] = Distributor(environment, ssn_reader, "customers")
+    distributors["products"] = Distributor(environment, product_reader, "products")
+
+
+class MyUser(HttpUser):
+    host = "http://www.example.com"
+
+    @task
+    def my_task(self) -> None:
+        customer: Dict = next(distributors["customers"])
+        product: List[str] = next(distributors["products"])
+        self.client.get(f"/?customer={customer['ssn']}&product={product[0]}")
+
+
+if __name__ == "__main__":
+    run_single_user(MyUser)

locust-plugins-4.3.1/examples/products.csv

@@ -0,0 +1,2 @@
+asdf1
+qwerty2

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/locust_plugins/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '4.2.0'
-__version_tuple__ = version_tuple = (4, 2, 0)
+__version__ = version = '4.3.1'
+__version_tuple__ = version_tuple = (4, 3, 1)

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/locust_plugins/csvreader.py

@@ -1,7 +1,8 @@
 import csv
+from typing import Iterator, Dict
 
 
-class CSVReader:
+class CSVReader(Iterator):
     "Read test data from csv file using an iterator"
 
     def __init__(self, file, **kwargs):
@@ -21,8 +22,8 @@ class CSVReader:
             return next(self.reader)
 
 
-class CSVDictReader:
-    "Read test data from csv file using an iterator"
+class CSVDictReader(Iterator[Dict]):
+    "Read test data from csv file using an iterator, returns rows as dicts"
 
     def __init__(self, file, **kwargs):
         try:

locust-plugins-4.3.1/locust_plugins/distributor.py

@@ -0,0 +1,59 @@
+from typing import Dict, Iterator, Optional
+import logging
+from gevent.event import AsyncResult
+import greenlet
+from locust.env import Environment
+from locust.runners import WorkerRunner
+
+_results: Dict[int, AsyncResult] = {}
+
+
+class Distributor:
+    def __init__(self, environment: Environment, iterator: Optional[Iterator], name="distributor"):
+        """Register distributor method handlers and tie them to use the iterator that you pass.
+
+        iterator is not used on workers, so you can leave it as None there.
+        """
+        self.iterator = iterator
+        self.name = name
+        self.runner = environment.runner
+        assert iterator or isinstance(
+            self.runner, WorkerRunner
+        ), "iterator is a mandatory parameter when not on a worker runner"
+        if self.runner:
+            # received on master
+            def _distributor_request(environment: Environment, msg, **kwargs):
+                item = next(self.iterator)
+                self.runner.send_message(
+                    f"_{name}_response",
+                    {"item": item, "gid": msg.data["gid"]},
+                    client_id=msg.data["client_id"],
+                )
+
+            # received on worker
+            def _distributor_response(environment: Environment, msg, **kwargs):
+                _results[msg.data["gid"]].set(msg.data)
+
+            self.runner.register_message(f"_{name}_request", _distributor_request)
+            self.runner.register_message(f"_{name}_response", _distributor_response)
+
+    def __next__(self):
+        """Get the next data dict from iterator
+
+        Args:
+            user (User): current user object (we use the object id of the User to keep track of who's waiting for which data)
+        """
+        if not self.runner:  # no need to do anything clever if there is no runner
+            assert self.iterator
+            return next(self.iterator)
+
+        gid = greenlet.getcurrent().minimal_ident  # type: ignore
+
+        if gid in _results:
+            logging.warning("This user was already waiting for data. Strange.")
+
+        _results[gid] = AsyncResult()
+        self.runner.send_message(f"_{self.name}_request", {"gid": gid, "client_id": self.runner.client_id})
+        item = _results[gid].get()["item"]  # this waits for the reply
+        del _results[gid]
+        return item

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/locust_plugins/mongoreader.py

@@ -1,23 +1,71 @@
-from typing import Iterator
+from typing import Iterator, Optional, Dict
 from pymongo import MongoClient
-import pymongo.collection
+from pymongo.collection import Collection
+from pymongo.cursor import Cursor
 from datetime import datetime, timezone
 import logging
 import time
 from contextlib import contextmanager
 import os
+from os import environ as env
 from abc import ABC, abstractmethod
 from gevent.lock import Semaphore
 
 dblock = Semaphore()
 
 
+class MongoLRUReader(Iterator[Dict]):
+    def __init__(
+        self,
+        filter: Dict,  # pylint: disable=redefined-builtin
+        timestamp_field: str,
+        coll: Optional[Collection] = None,
+    ):
+        """A thread safe iterator to read test data from a mongo collection sorted by least-recently-used.
+
+        Order is ensured even if Locust is restarted, because the timestamp_field is updated on every iteration step.
+
+        Iteration is quite fast, but the first query can be slow if you dont have an index that covers the filter and sort fields.
+
+        Args:
+            filter (Dict): Query filter statement
+            sort_field (str): Time stamp field, e.g. "last_used"
+            collection (pymongo.collection.Collection, optional): By default, we use LOCUST_MONGO, LOCUST_MONGO_DATABASE and LOCUST_MONGO_COLLECTION env vars to get the collection, but you can also pass a pre-existing Collection.
+
+        """
+        self.timestamp_field = timestamp_field
+        self.coll: Collection = (
+            coll or MongoClient(env["LOCUST_MONGO"])[env["LOCUST_MONGO_DATABASE"]][env["LOCUST_MONGO_COLLECTION"]]
+        )
+        self.cursor: Cursor = self.coll.find(filter, sort=[(self.timestamp_field, 1)])
+        records_in_buffer = self.cursor._refresh()  # trigger fetch immediately instead of waiting for the first next()
+        if not records_in_buffer:
+            logging.warning(f"No records returned from query {filter}")
+
+    def __next__(self) -> dict:
+        try:
+            with dblock:
+                doc: dict = next(self.cursor)
+            self.coll.find_one_and_update(
+                {"_id": doc["_id"]},
+                {"$set": {self.timestamp_field: datetime.now(tz=timezone.utc)}},
+            )
+            return doc
+        except StopIteration:
+            with dblock:
+                self.cursor.rewind()
+            return next(self.cursor)
+
+
+### Legacy
+
+
 class NoUserException(Exception):
     pass
 
 
 class User(dict):
-    def __init__(self, coll: pymongo.collection.Collection, query: dict):
+    def __init__(self, coll: Collection, query: dict):
         self.coll = coll
         with dblock:
             data = self.coll.find_one_and_update(
@@ -55,6 +103,7 @@ class MongoReader(AbstractReader):
         self.reduced_filters = []
         self.delay_warning = 0
         self.query = {"$and": filters + [{"logged_in": False}]}
+        logging.warning("MongoReader is deprecated, please switch to MongoReaderLRU")
 
     @contextmanager
     def user(self, query: dict = None):

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/locust_plugins/users/socketio.py

@@ -88,7 +88,7 @@ class SocketIOUser(User):
             logging.debug(f"WSR: {message}")
             self.on_message(message)
 
-    def send(self, body, name=None, context={}):
+    def send(self, body, name=None, context={}, opcode=websocket.ABNF.OPCODE_TEXT):
         if not name:
             if body == "2":
                 name = "2 heartbeat"
@@ -112,7 +112,7 @@ class SocketIOUser(User):
             context={**self.context(), **context},
         )
         logging.debug(f"WSS: {body}")
-        self.ws.send(body)
+        self.ws.send(body, opcode)
 
     def sleep_with_heartbeat(self, seconds):
         while seconds >= 0:

{locust-plugins-4.2.0 → locust-plugins-4.3.1/locust_plugins.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: locust-plugins
-Version: 4.2.0
+Version: 4.3.1
 Summary: Useful plugins/extensions for Locust
 Home-page: https://github.com/SvenskaSpel/locust-plugins
 Author: Lars Holmberg
@@ -16,20 +16,44 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Requires-Python: >=3.8, <4
+License-File: LICENSE
+Requires-Dist: locust>=2.20.0
+Requires-Dist: typing-extensions
 Provides-Extra: websocket
+Requires-Dist: websocket-client; extra == "websocket"
 Provides-Extra: playwright
+Requires-Dist: playwright>=1.34.0; extra == "playwright"
 Provides-Extra: dashboards
+Requires-Dist: psycogreen; extra == "dashboards"
+Requires-Dist: psycopg2-binary; extra == "dashboards"
 Provides-Extra: kafka
+Requires-Dist: confluent-kafka; extra == "kafka"
 Provides-Extra: mongo
+Requires-Dist: pymongo; extra == "mongo"
 Provides-Extra: mqtt
+Requires-Dist: paho-mqtt>=1.5.0; extra == "mqtt"
 Provides-Extra: appinsights
+Requires-Dist: opencensus-ext-azure; extra == "appinsights"
 Provides-Extra: resource
+Requires-Dist: lxml; extra == "resource"
 Provides-Extra: boto3
+Requires-Dist: boto3; extra == "boto3"
 Provides-Extra: all
+Requires-Dist: websocket-client; extra == "all"
+Requires-Dist: playwright>=1.34.0; extra == "all"
+Requires-Dist: psycogreen; extra == "all"
+Requires-Dist: psycopg2-binary; extra == "all"
+Requires-Dist: confluent-kafka; extra == "all"
+Requires-Dist: pymongo; extra == "all"
+Requires-Dist: paho-mqtt>=1.5.0; extra == "all"
+Requires-Dist: opencensus-ext-azure; extra == "all"
+Requires-Dist: lxml; extra == "all"
+Requires-Dist: boto3; extra == "all"
 Provides-Extra: webdriver
-License-File: LICENSE
+Requires-Dist: selenium>=4.0.0; extra == "webdriver"
 
 https://github.com/SvenskaSpel/locust-plugins

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/locust_plugins.egg-info/SOURCES.txt

@@ -20,6 +20,7 @@ examples/cmd_line_examples.sh
 examples/connection_pool_ex.py
 examples/constant_total_ips_ex.py
 examples/csvreader_ex.py
+examples/distributor_ex.py
 examples/embedded_resource_manager_ex.py
 examples/jmeter_listener_example.py
 examples/kafka_ex.py
@@ -27,6 +28,7 @@ examples/mongoreader_ex.py
 examples/mqtt_ex.py
 examples/playwright-recording.py
 examples/playwright_ex.py
+examples/products.csv
 examples/reschedule_on_fail_ex.py
 examples/rest_ex.py
 examples/socketio_ex.py
@@ -42,6 +44,7 @@ locust_plugins/__init__.py
 locust_plugins/_version.py
 locust_plugins/connection_pools.py
 locust_plugins/csvreader.py
+locust_plugins/distributor.py
 locust_plugins/mongoreader.py
 locust_plugins/py.typed
 locust_plugins/transaction_manager.py

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/locust_plugins.egg-info/requires.txt

@@ -1,4 +1,4 @@
-locust>=2.16.1
+locust>=2.20.0
 typing-extensions
 
 [all]

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/setup.py

@@ -32,6 +32,7 @@ setup(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Intended Audience :: Developers",
         "Intended Audience :: System Administrators",
     ],
@@ -45,7 +46,7 @@ setup(
     package_data={"locust_plugins": ["py.typed"]},
     zip_safe=False,
     install_requires=[
-        "locust>=2.16.1",
+        "locust>=2.20.0",
         "typing-extensions",
     ],
     extras_require={

{locust-plugins-4.2.0 → locust-plugins-4.3.1}/tox.ini

@@ -32,6 +32,10 @@ commands =
     grep -m 1 'Manual' output.txt
     grep -m 1 'LCP' output.txt
     bash -ec "! grep 'object has no attribute' output.txt"
+    bash -ec "(cd examples && PYTHONUNBUFFERED=1 locust -f distributor_ex.py --headless -t 5 -u 4 --processes 4) |& tee output.txt || true"
+    grep -m 1 'customer=2099010101-1111' output.txt
+    grep -m 1 'product=asdf1' output.txt
+    bash -ec "! grep -q Traceback output.txt"
     locust -f examples/jmeter_listener_example.py --headless -t 1
     locust-compose up -d
     sleep 15