mite 2.5.2__tar.gz → 2.5.4__tar.gz

{mite-2.5.2 → mite-2.5.4}/Dockerfile.perftest

@@ -4,7 +4,7 @@ COPY ./test/perf/test-requirements.txt /test-requirements.txt
 
 RUN pip install -r test-requirements.txt
 
-RUN pip install bs4 docopt flask msgpack-python nanomsg pyzmq uvloop
+RUN pip install bs4 docopt flask msgpack-python nanomsg pyzmq uvloop==0.21.0
 
 COPY . /package
 

{mite-2.5.2 → mite-2.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mite
-Version: 2.5.2
+Version: 2.5.4
 Summary: A Python Performance Testing Framework
 Home-page: https://github.com/sky-uk/mite/
 Classifier: License :: OSI Approved :: MIT License
@@ -21,7 +21,7 @@ Requires-Dist: nanomsg
 Requires-Dist: prettytable
 Requires-Dist: pyzmq
 Requires-Dist: selenium<4
-Requires-Dist: uvloop
+Requires-Dist: uvloop==0.21.0
 Requires-Dist: websockets
 Provides-Extra: amqp
 Requires-Dist: aio_pika; extra == "amqp"

mite-2.5.4/cd-scripts/cdBuild.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+# Because of the settings of the environment where the pipeline runs, cdBuild.sh is used for both CI testing and manual releases as we can change the default scripts used.
+# In a normal situation, we would have two separate scripts, one for CI and one for manual releases.
+# Here we check the JOB_NAME variable to determine which kind of run it is and call the appropriate script.
+
+# manual-release job will run the release script directly
+# any other job (like miteci-release) will run the test script. Then the pipeline shape will take care of calling the release script if the tests pass for the CI job.
+
+echo $JOB_NAME
+
+if [[ "$JOB_NAME" =~ "mite-ci" ]]; then
+    echo "##### Job running as CI. Proceeding with the Test script. ######"
+    ./cd-scripts/runTests.sh
+fi

mite-2.5.2/cd-scripts/cdRelease.py → mite-2.5.4/cd-scripts/cdTagRelease.py

@@ -83,6 +83,12 @@ def create_and_push_tag(repo, tag):
 
 def parse_pr(pr_number):
     resp = requests.get(f"https://api.github.com/repos/sky-uk/mite/pulls/{pr_number}")
+    if resp.status_code != 200:
+        logger.error(
+            f"Request to GitHub not successfull: {resp.status_code} - {resp.text}"
+        )
+        sys.exit(1)
+
     pr_message = resp.json()["body"]
 
     matches = re.findall(

mite-2.5.2/cd-scripts/cdTagRelease.sh → mite-2.5.4/cd-scripts/lib.sh

@@ -1,10 +1,25 @@
 #!/bin/bash
 
+function isOnMaster() {
+	current_revision=$(git rev-parse HEAD)
+	branch=$(git branch -r --contains $current_revision)
+	set +e
+	echo "$branch" | grep -q "origin/master$"
+	result=$?
+	set -e
+	return ${result}
+}
+
 function tagBuild() {   
+    if [ -z "$1" ]; then
+        VERSION_INCREMENT_TYPE=""
+    else
+        VERSION_INCREMENT_TYPE="--${1}"
+    fi
     git config user.email "mite@noreply.github.com"
     git config user.name "Jenkins-CI"
     pip3 install docopt GitPython packaging requests
-    python3 cd-scripts/cdRelease.py
+    python3 cd-scripts/cdTagRelease.py ${VERSION_INCREMENT_TYPE}
     if (( $? == 1)); then
         VERSION_INCREMENT=false
     fi
@@ -49,25 +64,4 @@ function initPypirc() {
     echo -e "[pypi]" > ~/.pypirc
     echo -e "username = __token__" >> ~/.pypirc
     echo -e "password = $PYPI_TOKEN" >> ~/.pypirc
-}
-
-###### MAIN SECTION ######
-
-echo "##### Starting tag process and check for version increment #####"
-tagBuild
-
-echo "##### Look for Acurl changes #####"
-checkAcurl
-
-echo "##### Build Linux-Wheels #####"
-buildLinuxWheels
-
-if [ "$VERSION_INCREMENT" = false ]; then
-        echo "Mite version not incremented, nothing to upload"
-else
-    echo "##### Init .pypirc #####"
-    initPypirc
-
-    echo "##### Upload package #####"
-    python3 -m twine upload wheelhouse/*
-fi
+}

mite-2.5.4/cd-scripts/manualRelease.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+VERSION_INCREMENT_TYPE=${1}
+
+if isOnMaster; then
+    echo "##### Job running on MASTER. Proceeding with the Tag and Release script. ######"
+    echo "##### Starting tag process and check for version increment #####"
+    # We are passing over the VERSION_TYPE argument to tagBuild which will set the type of increment for the new version
+    # If not passed, cdRelease.py will use the  GH PR to check which kind of increment it is supposed to be as VERSION_TYPE will be an empty string
+    tagBuild ${VERSION_INCREMENT_TYPE}
+
+    echo "##### Look for Acurl changes #####"
+    checkAcurl
+
+    echo "##### Build Linux-Wheels #####"
+    buildLinuxWheels
+
+    if [ "$VERSION_INCREMENT" = false ]; then
+            echo "Mite version not incremented, nothing to upload"
+    else
+        echo "##### Init .pypirc #####"
+        initPypirc
+
+        echo "##### Upload package #####"
+        python3 -m twine upload wheelhouse/*
+    fi
+
+else
+	echo "Not going to build the mite image because the job is not running on the master branch"
+fi

mite-2.5.4/cd-scripts/runTests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+
+echo "##### Run pre-commit checks #####"
+/home/jenkins/.local/bin/pre-commit run --origin HEAD --source origin/master
+PRE_COMMIT_STATUS=$?
+
+if [ $PRE_COMMIT_STATUS -ne 0 ]; then
+    git diff
+fi
+
+echo "##### Run tests with tox #####"
+tox; TOX_EXIT_CODE=$?
+[ "$TOX_EXIT_CODE" -eq 0 -a "$PRE_COMMIT_STATUS" -eq 0 ] || exit 1
+echo "##### Pre-commit and Tests passed #####"

mite-2.5.4/cd.yml

@@ -0,0 +1,49 @@
+version: 1
+
+# Further ideas for jobs to run:
+# - license check
+# - make sure test coverage increases
+# - merge all tox tests into one using pyenv
+# And, once we're sure that the pipeline is working:
+# - integration/performance tests
+# Once we have docs:
+# - documentation coverage
+# - docs build (on master only)
+
+context: sky-identity
+triggering: multibranch
+
+defaultNodes:
+  default: sky-identity-nonprod-mite-slave-buildkit
+
+modules:
+  mite-ci:
+    pipelineShape: simple
+    directory: "/"
+    htmlReports:
+      - name: Coverage Report
+        directory: coverage_html_report
+        files: ["index.html"]
+
+
+  mite-manual-release:
+    pipelineShape: simple
+    defaultTaskExecution: ./cd-scripts/manualRelease.sh
+    phaseLocks:
+      cdBuild:
+        defaultLockEnabled: false
+    parameters:
+      main:
+        - name: VERSION_INCREMENT_TYPE
+          type: choice
+          description: The version segment that you want increment for the release
+          options: [patch, minor, major]
+
+  # TODO find a work around for the issue with combining build and release in one pipeline
+  # mite-ci-and-release:
+  #   pipelineShape: build-release
+  #   directory: "/"
+  #   htmlReports:
+  #     - name: Coverage Report
+  #       directory: coverage_html_report
+  #       files: ["index.html"]

{mite-2.5.2 → mite-2.5.4}/dev-requirements.txt

@@ -5,3 +5,4 @@ pre-commit~=2.20.0
 flake8~=3.7.8
 black~=24.3.0
 coverage~=5.1
+sqlalchemy

{mite-2.5.2 → mite-2.5.4}/mite.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mite
-Version: 2.5.2
+Version: 2.5.4
 Summary: A Python Performance Testing Framework
 Home-page: https://github.com/sky-uk/mite/
 Classifier: License :: OSI Approved :: MIT License
@@ -21,7 +21,7 @@ Requires-Dist: nanomsg
 Requires-Dist: prettytable
 Requires-Dist: pyzmq
 Requires-Dist: selenium<4
-Requires-Dist: uvloop
+Requires-Dist: uvloop==0.21.0
 Requires-Dist: websockets
 Provides-Extra: amqp
 Requires-Dist: aio_pika; extra == "amqp"

{mite-2.5.2 → mite-2.5.4}/mite.egg-info/SOURCES.txt

@@ -25,9 +25,11 @@ tox.ini
 .github/pull_request_template.md
 .github/ISSUE_TEMPLATE/bug_report.md
 .github/ISSUE_TEMPLATE/feature_request.md
-cd-scripts/cdBuild-1.sh
-cd-scripts/cdRelease.py
-cd-scripts/cdTagRelease.sh
+cd-scripts/cdBuild-old.sh
+cd-scripts/cdBuild.sh
+cd-scripts/cdTagRelease.py
+cd-scripts/lib.sh
+cd-scripts/manualRelease.sh
 cd-scripts/runTests.sh
 docs/Makefile
 docs/components.rst
@@ -89,6 +91,10 @@ mite/web/__init__.py
 mite/web/prometheus.py
 mite_amqp/__init__.py
 mite_browser/__init__.py
+mite_db_datapool/README.md
+mite_db_datapool/__init__.py
+mite_db_datapool/db_to_pool.py
+mite_db_datapool/mite_db_example.py
 mite_finagle/README.md
 mite_finagle/__init__.py
 mite_finagle/mux.py
@@ -123,6 +129,7 @@ test/test_config.py
 test/test_context.py
 test/test_controller.py
 test/test_datapools.py
+test/test_db_to_pool.py
 test/test_har_to_mite.py
 test/test_logging.py
 test/test_mite_amqp.py

{mite-2.5.2 → mite-2.5.4}/mite.egg-info/requires.txt

@@ -7,7 +7,7 @@ nanomsg
 prettytable
 pyzmq
 selenium<4
-uvloop
+uvloop==0.21.0
 websockets
 
 [amqp]

{mite-2.5.2 → mite-2.5.4}/mite.egg-info/top_level.txt

@@ -1,6 +1,7 @@
 mite
 mite_amqp
 mite_browser
+mite_db_datapool
 mite_finagle
 mite_http
 mite_kafka

mite-2.5.4/mite_db_datapool/README.md

@@ -0,0 +1,45 @@
+# Database -> DataPool for Mite 
+
+## Overview
+
+This module provides database-backed data pools for Mite testing scenarios. It allows you to load test data directly from databases and use it in your load testing journeys.
+
+## Why SQLAlchemy?
+
+We chose **SQLAlchemy** over **pyodbc** for several reasons:
+
+- **Database Agnostic**: Works with SQLite, PostgreSQL, MySQL, SQL Server, Oracle, etc.
+- **Connection Pooling**: Built-in connection management and pooling
+- **Simpler API**: Clean, Pythonic interface for database operations  
+- **Better Error Handling**: More informative error messages
+- **No Driver Dependencies**: SQLAlchemy handles driver selection automatically
+- **URL-based Configuration**: Easy connection string format
+
+## Files
+
+### `db_to_pool.py`
+Core implementation containing:
+- `DBIterableDataPool`: Load data once, consume sequentially (burned data)
+- `DBRecyclableIterableDataPool`: Load data once, reuse infinitely
+- Memory protection with `max_size` parameter to prevent OOM with large datasets
+
+### `mite_db_example.py`  
+Working example showing:
+- SQLite database creation with test data
+- DataPool setup and configuration
+- Integration with Mite journey functions
+- Basic load testing scenario
+
+
+## Testing
+
+Run the example:
+```bash
+# From mite directory
+python mite_db_datapool/mite_db_example.py
+```
+
+Run unit tests:
+```bash
+pytest test/test_db_to_pool.py -v
+```

mite-2.5.4/mite_db_datapool/__init__.py

@@ -0,0 +1,8 @@
+# mite_db_datapool package
+from .db_to_pool import (
+    DataPoolExhausted,
+    DBIterableDataPool,
+    DBRecyclableIterableDataPool,
+)
+
+__all__ = ["DBIterableDataPool", "DBRecyclableIterableDataPool", "DataPoolExhausted"]

mite-2.5.4/mite_db_datapool/db_to_pool.py

@@ -0,0 +1,95 @@
+import logging
+from collections import deque
+
+from sqlalchemy import text
+
+from mite.datapools import DataPoolExhausted, DataPoolItem
+
+logger = logging.getLogger(__name__)
+
+
+class DBIterableDataPool:
+    def __init__(
+        self,
+        db_engine,
+        query,
+        preload_minimum=None,
+        max_size=None,
+    ):
+        self.db_engine = db_engine
+        self.query = query
+        self.max_size = max_size
+        self._data = []
+        self.item_index = 0
+        self.db_exhausted = False
+        self.preload_minimum = preload_minimum
+
+        # Adjust preload_minimum if needed
+        if max_size and preload_minimum and preload_minimum > max_size:
+            self.preload_minimum = max_size
+
+        # Load initial data
+        if preload_minimum:
+            while len(self._data) < self.preload_minimum and not self.db_exhausted:
+                self.populate()
+        else:
+            while not self.db_exhausted:
+                self.populate()
+
+    def populate(self):
+        # Stop if at max_size
+        if self.max_size is not None and len(self._data) >= self.max_size:
+            return
+
+        with self.db_engine.connect() as conn:
+            result = conn.execute(text(self.query))
+            rows = result.fetchall()
+
+            if not rows:
+                self.db_exhausted = True
+                return
+
+            # Add rows up to max_size
+            rows_added = 0
+            for row in rows:
+                if self.max_size is not None and len(self._data) >= self.max_size:
+                    break
+                self._data.append((self.item_index, dict(row._mapping)))
+                self.item_index += 1
+                rows_added += 1
+
+            # Only mark as exhausted if we processed all available rows
+            # If we only processed some rows due to max_size limit, don't mark as exhausted
+            if rows_added == len(rows):
+                self.db_exhausted = True
+
+    async def checkout(self, config):
+        try:
+            item_id, row = self._data.pop(0)
+            return DataPoolItem(item_id, row)
+        except IndexError as e:
+            raise DataPoolExhausted() from e
+
+    async def checkin(self, item_id):
+        pass
+
+
+class DBRecyclableIterableDataPool(DBIterableDataPool):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._available = deque(range(len(self._data)))
+        self._checked_out = set()
+
+    async def checkout(self, config):
+        if self._available:
+            item_id = self._available.popleft()
+            self._checked_out.add(item_id)
+            row = self._data[item_id][1]
+            return DataPoolItem(item_id, row)
+        else:
+            raise DataPoolExhausted("Recyclable iterable datapool was emptied!")
+
+    async def checkin(self, item_id):
+        if item_id in self._checked_out:
+            self._checked_out.remove(item_id)
+            self._available.append(item_id)

mite-2.5.4/mite_db_datapool/mite_db_example.py

@@ -0,0 +1,39 @@
+import sqlite3
+
+from db_to_pool import DBIterableDataPool
+from sqlalchemy import create_engine
+
+from mite import ensure_average_separation
+
+# Step 1: Create a test SQLite database and table (local file)
+DB_PATH = "example_test.db"
+conn = sqlite3.connect(DB_PATH)
+conn.execute("CREATE TABLE IF NOT EXISTS test_table (id INTEGER PRIMARY KEY, value TEXT)")
+conn.execute("DELETE FROM test_table")
+conn.execute("INSERT INTO test_table (value) VALUES ('mite1'), ('mite2'), ('mite3')")
+conn.commit()
+conn.close()
+
+# Step 2: Set up SQLAlchemy engine (same file)
+engine = create_engine(f"sqlite:///{DB_PATH}")
+
+# Step 3: Test DBIterableDataPool
+db_datapool = DBIterableDataPool(
+    engine, "SELECT * FROM test_table", preload_minimum=3, max_size=2
+)
+
+
+async def db_journey(ctx, item_id, row):
+    async with ensure_average_separation(1):
+        print("running journey", item_id, row)
+
+
+def scenario():
+    return [["db_folder:db_journey", db_datapool, lambda s, e: 1]]
+
+
+# Simple test to see it working
+if __name__ == "__main__":
+    print(f"✓ DataPool created with {len(db_datapool._data)} items")
+    for i, (item_id, data) in enumerate(db_datapool._data):
+        print(f"  Item {i}: {data}")

{mite-2.5.2 → mite-2.5.4}/setup.cfg

@@ -25,7 +25,7 @@ install_requires =
 	prettytable
 	pyzmq
 	selenium<4
-	uvloop
+	uvloop==0.21.0
 	websockets
 setup_requires = pytest-runner
 packages = find:

mite-2.5.4/test/test_db_to_pool.py

@@ -0,0 +1,155 @@
+import sqlite3
+
+import pytest
+from sqlalchemy import create_engine
+
+from mite_db_datapool.db_to_pool import (
+    DataPoolExhausted,
+    DBIterableDataPool,
+    DBRecyclableIterableDataPool,
+)
+
+
+@pytest.fixture
+def setup_test_db():
+    """Set up a test SQLite database and table"""
+    conn = sqlite3.connect("testdb.sqlite")
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS test_table (id INTEGER PRIMARY KEY, value TEXT)"
+    )
+    conn.execute("DELETE FROM test_table")
+    conn.execute("INSERT INTO test_table (value) VALUES ('foo'), ('bar'), ('baz')")
+    conn.commit()
+    conn.close()
+
+    engine = create_engine("sqlite:///testdb.sqlite")
+    yield engine
+
+    # Cleanup
+    import os
+
+    if os.path.exists("testdb.sqlite"):
+        os.remove("testdb.sqlite")
+
+
+@pytest.fixture
+def setup_large_test_db():
+    """Set up a test SQLite database with more data for max_size testing"""
+    conn = sqlite3.connect("testdb_large.sqlite")
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS large_table (id INTEGER PRIMARY KEY, value TEXT)"
+    )
+    conn.execute("DELETE FROM large_table")
+
+    # Insert 100 rows for testing max_size functionality
+    values = [(f"item_{i}",) for i in range(100)]
+    conn.executemany("INSERT INTO large_table (value) VALUES (?)", values)
+    conn.commit()
+    conn.close()
+
+    engine = create_engine("sqlite:///testdb_large.sqlite")
+    yield engine
+
+    # Cleanup
+    import os
+
+    if os.path.exists("testdb_large.sqlite"):
+        os.remove("testdb_large.sqlite")
+
+
+@pytest.mark.asyncio
+async def test_db_iterable_datapool(setup_test_db):
+    """Test basic DBIterableDataPool functionality"""
+    engine = setup_test_db
+    pool = DBIterableDataPool(engine, "SELECT * FROM test_table", preload_minimum=3)
+
+    checked_out_items = []
+    try:
+        while True:
+            item = await pool.checkout(config={})
+            checked_out_items.append(item)
+    except DataPoolExhausted:
+        pass
+
+    assert len(checked_out_items) == 3
+
+
+@pytest.mark.asyncio
+async def test_db_recyclable_iterable_datapool(setup_test_db):
+    """Test basic DBRecyclableIterableDataPool functionality"""
+    engine = setup_test_db
+    recycle_pool = DBRecyclableIterableDataPool(
+        engine, "SELECT * FROM test_table", preload_minimum=3
+    )
+
+    # Checkout items
+    items = []
+    for _ in range(3):
+        item = await recycle_pool.checkout(config={})
+        items.append(item.id)
+
+    # Check them back in
+    for item_id in items:
+        await recycle_pool.checkin(item_id)
+
+    # Try to checkout again - should work since items are recycled
+    recycled_items = []
+    for _ in range(3):
+        item = await recycle_pool.checkout(config={})
+        recycled_items.append(item.id)
+
+    assert len(recycled_items) == 3
+
+
+def test_max_size_limits_data_loading(setup_large_test_db):
+    """Test that max_size prevents OOM with large datasets"""
+    engine = setup_large_test_db
+
+    pool = DBIterableDataPool(
+        engine,
+        "SELECT * FROM large_table",
+        preload_minimum=3,  # Wants to load 3 items
+        max_size=2,  # Limit to 2 items even though 100 are available
+    )
+
+    assert len(pool._data) == 2
+
+
+def test_infinite_loop_prevention(setup_large_test_db):
+    """Test that preload_minimum > max_size doesn't cause infinite loop (critical fix)"""
+    import time
+
+    start_time = time.time()
+
+    pool = DBIterableDataPool(
+        setup_large_test_db,
+        "SELECT * FROM large_table",
+        preload_minimum=50,  # Want 50
+        max_size=10,  # Limited to 10
+    )
+
+    end_time = time.time()
+
+    assert end_time - start_time < 1.0  # Should complete quickly
+    assert pool.preload_minimum == 10  # Auto-adjusted
+    assert len(pool._data) == 10
+
+
+@pytest.mark.asyncio
+async def test_recyclable_with_max_size(setup_large_test_db):
+    """Test recyclable pool respects max_size"""
+    engine = setup_large_test_db
+
+    recycle_pool = DBRecyclableIterableDataPool(
+        engine, "SELECT * FROM large_table", preload_minimum=3, max_size=2
+    )
+
+    assert len(recycle_pool._data) == 2
+    assert len(recycle_pool._available) == 2
+
+    # Test checkout/checkin cycle
+    item = await recycle_pool.checkout(config={})
+    assert len(recycle_pool._available) == 1
+
+    await recycle_pool.checkin(item.id)
+    assert len(recycle_pool._available) == 2

{mite-2.5.2 → mite-2.5.4}/test-requirements.txt

@@ -14,5 +14,7 @@ pytest-httpbin
 pytest-httpserver
 sanic
 werkzeug==3.1.3
-uvloop~=0.19.0
-setuptools>=78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability
+sqlalchemy
+uvloop==0.21.0
+setuptools>=78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability
+

mite-2.5.2/cd-scripts/runTests.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-function isOnMaster() {
-    current_revision=$(git rev-parse HEAD)
-    branch=$(git branch -r --contains $current_revision)
-    set +e
-    echo "$branch" | grep -q "origin/master$"
-    result=$?
-    set -e
-    return ${result}
-}
-
-
-echo "##### Run pre-commit checks #####"
-/home/jenkins/.local/bin/pre-commit run --origin HEAD --source origin/master
-PRE_COMMIT_STATUS=$?
-
-if [ $PRE_COMMIT_STATUS -ne 0 ]; then
-    git diff
-fi
-
-echo "##### Run tests with tox #####"
-tox; TOX_EXIT_CODE=$?
-[ "$TOX_EXIT_CODE" -eq 0 -a "$PRE_COMMIT_STATUS" -eq 0 ] || exit 1
-
-if isOnMaster ; then
-    echo "##### Job running on MASTER. Proceeding with the Tag and Release script. ######"
-    ./cd-scripts/cdTagRelease.sh
-else
-    echo "###### Job running on a Branch. Stopping here. ######"
-fi