pytest-fly 0.0.1__py3-none-any.whl

pytest_fly/__init__.py

@@ -0,0 +1 @@
+from .plugin import pytest_addoption, pytest_runtest_logreport, pytest_sessionfinish

pytest_fly/__main__.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+from ismain import is_main
+
+from .visualization import visualize
+
+
+def main():
+    visualize(Path("temp", "ptest-fly.png"))  # todo: make output file a command line argument
+
+
+if is_main():
+    main()

pytest_fly/__version__.py

@@ -0,0 +1,2 @@
+author = "abel"
+application_name = "pytest-fly"

pytest_fly/db.py

@@ -0,0 +1,207 @@
+import sqlite3
+from pathlib import Path
+from msqlite import MSQLite
+import json
+import uuid
+from functools import cache
+from logging import getLogger
+import time
+from dataclasses import dataclass
+from collections import defaultdict
+
+from appdirs import user_data_dir
+
+from _pytest.reports import BaseReport
+
+from src.pytest_fly.report_converter import report_to_json
+
+from .__version__ import author, application_name
+
+g_table_name = ""  # will get set at initialization
+fly_db_path = Path(user_data_dir(application_name, author), f"{application_name}.db")
+
+log = getLogger(application_name)
+
+
+def set_db_path(db_path: Path | str):
+    global fly_db_path
+    fly_db_path = Path(db_path)
+
+
+def get_db_path() -> Path:
+    fly_db_path.parent.mkdir(parents=True, exist_ok=True)
+    return Path(fly_db_path)
+
+
+def set_table_name(table_name: str):
+    global g_table_name
+    g_table_name = table_name
+
+
+def get_table_name() -> str:
+    return g_table_name
+
+
+@cache
+def _get_process_guid() -> str:
+    """
+    Get a unique guid for this process by using functools.cache.
+    :return: GUID string
+    """
+    return str(uuid.uuid4())
+
+
+# "when" is a keyword in SQLite so use "pt_when"
+fly_schema = {"id PRIMARY KEY": int, "ts": float, "uid": str, "pt_when": str, "nodeid": str, "report": json}
+
+
+def get_table_name_from_report(report: BaseReport) -> str:
+    """
+    Get the table name from the report file path
+    """
+    table_name = Path(report.fspath).parts[0]
+    set_table_name(table_name)
+    return table_name
+
+
+def write_report(report: BaseReport):
+    """
+    Write a pytest report to the database
+    :param report: pytest report
+    """
+    try:
+        testrun_uid = report.testrun_uid  # pytest-xdist
+        is_xdist = True
+    except AttributeError:
+        testrun_uid = _get_process_guid()  # single threaded
+        is_xdist = False
+    table_name = get_table_name_from_report(report)
+    pt_when = report.when
+    node_id = report.nodeid
+    setattr(report, "is_xdist", is_xdist)  # signify if we're running pytest-xdist or not
+    db_path = get_db_path()
+    with MSQLite(db_path, table_name, fly_schema) as db:
+        report_json = report_to_json(report)
+        statement = f"INSERT OR REPLACE INTO {table_name} (ts, uid, pt_when, nodeid, report) VALUES ({time.time()}, '{testrun_uid}', '{pt_when}', '{node_id}', '{report_json}')"
+        try:
+            db.execute(statement)
+        except sqlite3.OperationalError as e:
+            log.error(f"{e}:{statement}")
+
+
+def read_json_objects_by_uid(uid: str, table_name: str = get_table_name()):
+    with MSQLite(get_db_path(), table_name) as db:
+        statement = f"SELECT * FROM {table_name} WHERE uid = {uid}"
+        rows = db.execute(statement)
+    return rows
+
+
+def get_all_test_run_ids(table_name: str = get_table_name()) -> set[str]:
+    with MSQLite(get_db_path(), table_name) as db:
+        test_run_ids = set()
+        rows = db.execute(f"SELECT * FROM {table_name}")
+        for row in rows:
+            run_id = row[2]
+            test_run_ids.add(run_id)
+    return test_run_ids
+
+
+meta_session_table_name = "_session"
+meta_session_schema = {"id PRIMARY KEY": int, "ts": float, "test_name": str, "state": str}
+
+
+def _get_most_recent_row_values(db) -> tuple[int | None, str | None, float | None]:
+    statement = f"SELECT * FROM {meta_session_table_name} ORDER BY ts DESC LIMIT 1"
+    rows = list(db.execute(statement))
+    row = rows[0] if len(rows) > 0 else None
+    if row is None:
+        id_state_ts = None, None, None
+    else:
+        id_state_ts = row[0], row[3], row[1]
+    return id_state_ts
+
+
+def write_start():
+    db_path = get_db_path()
+    with MSQLite(db_path, meta_session_table_name, meta_session_schema) as db:
+        # get the most recent state
+        id_value, state, ts = _get_most_recent_row_values(db)
+        if state != "start":
+            statement = f"INSERT OR REPLACE INTO {meta_session_table_name} (ts, state) VALUES ({time.time()}, 'start')"
+            db.execute(statement)
+
+
+def write_finish(test_name: str):
+    db_path = get_db_path()
+    with MSQLite(db_path, meta_session_table_name, meta_session_schema) as db:
+        id_value, state, ts = _get_most_recent_row_values(db)
+        now = time.time()
+        if state == "start":
+            statement = f"INSERT INTO {meta_session_table_name} (ts, test_name, state) VALUES ({now}, '{test_name}', 'finish')"
+        else:
+            statement = f"UPDATE {meta_session_table_name} SET ts = {now}, state = 'finish' WHERE id = {id_value}"
+        db.execute(statement)
+
+
+def get_most_recent_start_and_finish() -> tuple[str | None, float | None, float | None]:
+    db_path = get_db_path()
+    with MSQLite(db_path, meta_session_table_name, meta_session_schema) as db:
+        statement = f"SELECT * FROM {meta_session_table_name} ORDER BY ts DESC LIMIT 2"
+        rows = list(db.execute(statement))
+        start_ts = rows[1][1]
+        finish_ts = rows[0][1]
+        test_name = rows[0][2]
+    return test_name, start_ts, finish_ts
+
+
+@dataclass
+class RunInfo:
+    worker_id: str | None = None
+    start: float | None = None
+    stop: float | None = None
+    passed: bool | None = None
+
+
+@dataclass(frozen=True)
+class RunInfoKey:
+    test_id: str
+    when: str
+
+
+def get_most_recent_run_info() -> dict[str, dict[str, RunInfo]]:
+    test_name, start_ts, finish_ts = get_most_recent_start_and_finish()
+    db_path = get_db_path()
+    with MSQLite(db_path, test_name) as db:
+        statement = f"SELECT * FROM {test_name} WHERE ts >= {start_ts} and ts <= {finish_ts} ORDER BY ts"
+        rows = list(db.execute(statement))
+    run_infos = {}
+    for row in rows:
+        test_data = json.loads(row[-1])
+        test_id = test_data["nodeid"]
+        worker_id = test_data.get("worker_id")
+        when = test_data.get("when")
+        start = test_data.get("start")
+        stop = test_data.get("stop")
+        passed = test_data.get("passed")
+        if test_id in run_infos:
+            run_info = run_infos[test_id]
+            if start is not None:
+                if run_info[when].start is None:
+                    run_info[when].start = start
+                else:
+                    run_info[when].start = min(run_info[when].start, start)
+            if stop is not None:
+                if run_info[when].stop is None:
+                    run_info[when].stop = stop
+                else:
+                    run_info[when].stop = max(run_info[when].stop, stop)
+            if passed is not None:
+                run_info[when].passed = passed
+            if worker_id is not None:
+                run_info[when].worker_id = worker_id
+        else:
+            run_infos[test_id] = defaultdict(RunInfo)
+            run_infos[test_id][when] = RunInfo(worker_id, start, stop, passed)
+    # convert defaultdict to dict
+    run_infos = {test_id: dict(run_info) for test_id, run_info in run_infos.items()}
+    return run_infos

pytest_fly/plugin.py

@@ -0,0 +1,19 @@
+import pytest
+from _pytest.reports import BaseReport
+from src.pytest_fly.db import write_report, write_start, write_finish
+
+
+def pytest_addoption(parser):
+    write_start()
+    parser.addoption("--fly", action="store_true")
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_logreport(report: BaseReport):
+    write_report(report)
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_sessionfinish(session, exitstatus):
+    table_name = session.startpath.name
+    write_finish(table_name)

pytest_fly/report_converter.py

@@ -0,0 +1,67 @@
+from typing import Sized, Any
+import json
+import time
+import platform
+import getpass
+from functools import cache
+
+from _pytest.reports import BaseReport
+
+
+@cache
+def get_username() -> str:
+    """
+    Get the username of the current user
+    """
+    return getpass.getuser()
+
+
+@cache
+def get_computer_name() -> str:
+    """
+    Get the computer name
+    """
+    return platform.node()
+
+
+def _convert_report_to_dict(report: BaseReport) -> dict[str, Any]:
+    """
+    Convert a pytest Report to a dict, excluding zero-length iterables, None values, and other data that's not serializable. This isn't perfect but it seems to preserve
+    the data we're interested in.
+    :param report: Pytest Report
+    :return: a dict representation of the report
+    """
+    report_dict = {}
+    for attr in dir(report):
+        # Exclude private attributes and methods, None, and zero-length lists
+        if not attr.startswith("__") and (value := getattr(report, attr)) is not None:
+            has_size = isinstance(value, Sized)
+            if not has_size or has_size and len(value) > 0:
+                # Check if the attribute is serializable
+                try:
+                    json.dumps(value)
+                    report_dict[attr] = value
+                except TypeError:
+                    # a string representation starting with "<" generally isn't a useful serialization, so ignore it
+                    if not (s := str(value)).startswith("<"):
+                        report_dict[attr] = s
+    return report_dict
+
+
+def report_to_json(report: BaseReport) -> str:
+    """
+    Convert Pytest Report object to a JSON string (as much as possible)
+    :param report: Pytest Report
+    :return: JSON string representation of the report
+    """
+    d = _convert_report_to_dict(report)
+    # remove fields that we don't need that can have formatting issues with SQLite JSON
+    removes = ["sections", "capstdout", "capstderr"]
+    for remove in removes:
+        if remove in d:
+            del d[remove]
+    d["fly_timestamp"] = time.time()  # pytest-fly's own timestamp
+    d["username"] = get_username()
+    d["computer_name"] = get_computer_name()
+    s = json.dumps(d)
+    return s

pytest_fly/utilization.py

@@ -0,0 +1,58 @@
+from typing import List, Tuple, Dict
+from collections import defaultdict
+
+from .db import RunInfo
+
+
+def merge_intervals(intervals: List[Tuple[float, float]]) -> List[Tuple[float, float]]:
+    """Merge overlapping intervals."""
+    if not intervals:
+        return []
+
+    # Sort intervals by the start time
+    intervals.sort(key=lambda x: x[0])
+    merged = [intervals[0]]
+
+    for current_start, current_end in intervals[1:]:
+        last_end = merged[-1][1]
+
+        if current_start <= last_end:
+            merged[-1] = (merged[-1][0], max(last_end, current_end))
+        else:
+            merged.append((current_start, current_end))
+
+    return merged
+
+
+def calculate_utilization(data: Dict[str, Dict[str, RunInfo]]) -> Tuple[Dict[str, float], float]:
+    """
+    Calculate utilization for each worker and overall utilization.
+    :param data: Dictionary with test names as keys and dictionaries with phase names as keys and RunInfo objects as values.
+    :return: Tuple with a dictionary with worker IDs as keys and utilization as values and overall utilization.
+    """
+    worker_intervals = defaultdict(list)
+    all_times = []
+
+    # Collect intervals for each worker and all timestamps
+    for test in data.values():
+        for phase in test.values():
+            worker_intervals[phase.worker_id].append((phase.start, phase.stop))
+            all_times.extend([phase.start, phase.stop])
+
+    # Calculate total time span
+    total_time_span = max(all_times) - min(all_times)
+
+    # Calculate utilization for each worker
+    utilization = {}
+    total_busy_time_all_workers = 0
+    for worker_id, intervals in worker_intervals.items():
+        merged = merge_intervals(intervals)
+        total_busy_time = sum(end - start for start, end in merged)
+        total_busy_time_all_workers += total_busy_time  # Sum busy times for overall utilization
+        utilization[worker_id] = total_busy_time / total_time_span
+
+    # Calculate overall utilization
+    num_processors = len(worker_intervals)
+    overall_utilization = total_busy_time_all_workers / (total_time_span * num_processors)
+
+    return utilization, overall_utilization

pytest_fly/visualization.py

@@ -0,0 +1,85 @@
+import tkinter as tk
+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
+import matplotlib.pyplot as plt
+import numpy as np
+from typing import Dict
+from pathlib import Path
+
+from .db import get_most_recent_run_info, RunInfo
+from .utilization import calculate_utilization
+
+
+def visualize(plot_file_path: Path | None = None) -> None:
+    """
+    Visualize a timeline of test phases per worker.
+    :param plot_file_path: Path to save the plot to.
+    """
+    run_info = get_most_recent_run_info()
+
+    root = tk.Tk()
+    root.title("Test Phases Timeline")
+
+    fig, ax = plt.subplots(figsize=(20, 3 + len(run_info) * 0.5))  # Dynamic height based on the number of tests
+    _plot_timeline(run_info, fig, ax, plot_file_path)
+
+    canvas = FigureCanvasTkAgg(fig, master=root)  # A tk.DrawingArea
+    canvas.draw()
+    canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)
+
+    tk.mainloop()
+
+
+def _plot_timeline(data: Dict[str, Dict[str, RunInfo]], fig, ax, plot_file_path: Path | None) -> None:
+    """
+    Plot a timeline of test phases per worker.
+    :param data: Dictionary with test names as keys and dictionaries with phase names as keys and RunInfo objects as values.
+    :param fig: Matplotlib figure object.
+    :param ax: Matplotlib axis object.
+    :param plot_file_path: Path to save the plot to.
+    """
+    sorted_data = dict(sorted(data.items(), key=lambda x: x[0], reverse=True))
+    worker_utilization, overall_utilization = calculate_utilization(sorted_data)
+    earliest_start = min(phase.start for test in sorted_data.values() for phase in test.values())
+
+    workers = set(info.worker_id for test in sorted_data.values() for info in test.values())
+    colors = plt.cm.jet(np.linspace(0, 1, len(workers)))
+    worker_colors = dict(zip(workers, colors))
+
+    yticks, yticklabels = [], []
+    for i, (test_name, phases) in enumerate(sorted_data.items()):
+        for phase_name, phase_info in phases.items():
+            relative_start = phase_info.start - earliest_start
+            relative_stop = phase_info.stop - earliest_start
+            worker_id = phase_info.worker_id
+
+            ax.plot([relative_start, relative_stop], [i, i], color=worker_colors[worker_id], marker="o", markersize=4, label=worker_id if phase_name == "setup" else "")
+
+            if phase_name == list(phases.keys())[0]:
+                yticks.append(i)
+                yticklabels.append(test_name)
+
+    ax.set_yticks(yticks)
+    ax.set_yticklabels(yticklabels)
+    ax.set_xlabel("Time (seconds)")
+    ax.set_ylabel("Test Names")
+    ax.set_title("Timeline of Test Phases per Worker")
+    ax.grid(True)
+
+    ax.text(1.0, 1.02, f"Overall Utilization: {overall_utilization:.2%}", transform=ax.transAxes, horizontalalignment="right", fontsize=9)
+    text_position = 1.05
+    for worker, utilization in worker_utilization.items():
+        ax.text(1.0, text_position, f"{worker}: {utilization:.2%}", transform=ax.transAxes, horizontalalignment="right", fontsize=9)
+        text_position += 0.03
+
+    handles, labels = ax.get_legend_handles_labels()
+    by_label = dict(zip(labels, handles))
+    legend = ax.legend(by_label.values(), by_label.keys(), title="Workers", loc="upper left", bbox_to_anchor=(1.01, 1), fontsize=9)
+
+    plt.subplots_adjust(right=0.9)  # Make space for the legend
+
+    # Use bbox_extra_artists to include the legend in the tight layout calculation
+    if plot_file_path is not None:
+        plot_file_path.parent.mkdir(parents=True, exist_ok=True)
+        fig.savefig(plot_file_path, bbox_extra_artists=[legend])
+
+    # plt.show()

pytest_fly-0.0.1.dist-info/METADATA

@@ -0,0 +1,59 @@
+Metadata-Version: 2.3
+Name: pytest-fly
+Version: 0.0.1
+Summary: pytest observer
+Project-URL: Home, https://github.com/jamesabel/pytest-fly
+Project-URL: Repository, https://github.com/jamesabel/pytest-fly
+Author-email: James Abel <j@abel.co>
+Maintainer-email: James Abel <j@abel.co>
+License: 
+        The MIT License (MIT)
+        
+        Copyright (c) 2024 James Abel
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in
+        all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+        THE SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.12
+Requires-Dist: appdirs
+Requires-Dist: ismain
+Requires-Dist: msqlite
+Requires-Dist: pytest
+Description-Content-Type: text/markdown
+
+# pytest-fly
+
+pytest observer
+
+Installation
+------------
+
+You can install `pytest-fly` via `pip` from `PyPI`:
+
+```
+    pip install pytest-fly
+```

pytest_fly-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+pytest_fly/__init__.py,sha256=eIUdSiN4Fg5Rge-MozltxW8SaifDIO35mT9Ni6WQ8qI,86
+pytest_fly/__main__.py,sha256=UaJOPF1c6vlAuRFgxqsJvdPt5YMV9SBn0HmxH6G4jf0,240
+pytest_fly/__version__.py,sha256=Vu3vOZ_k30PP5uukW7uhpYxCgHXh9yYl-eKzlb2D0N4,50
+pytest_fly/db.py,sha256=dgwmy9Lb36W3yVkdrf1qRxGEdAfEJ9KMH3uaUz9fVJ8,7103
+pytest_fly/plugin.py,sha256=z1XAyS86a2uNFcC9M4iS2AETdO82t5PPVwgdSUJDWeg,501
+pytest_fly/report_converter.py,sha256=Zm-dRG8bhCHiCv4DgvKqO52pQsQr4gc-9y3NgSc1k1E,2253
+pytest_fly/utilization.py,sha256=pzKpNWpK_FF1Ocfj7fkd-tZvmc1Jx1X60x3nVbVJsHs,2198
+pytest_fly/visualization.py,sha256=DhaqOnzAUMTyNizVvgF0ayOJFTn7wziFnne-13UeYYk,3655
+pytest_fly-0.0.1.dist-info/METADATA,sha256=hhHg3k41Zb9ryGJElMVI34VPrsreMfE-JxZKuC4gPAk,2279
+pytest_fly-0.0.1.dist-info/WHEEL,sha256=as-1oFTWSeWBgyzh0O_qF439xqBe6AbBgt4MfYe5zwY,87
+pytest_fly-0.0.1.dist-info/entry_points.txt,sha256=V1GWjglijPbLnVLJ5o6y7fSh75rIbgoT4ShTkF53xic,35
+pytest_fly-0.0.1.dist-info/licenses/LICENSE,sha256=GRULexapWa5z0SwQbuxdiK3Kj-FPOWkXshLFGzkOWG4,1100
+pytest_fly-0.0.1.dist-info/RECORD,,

pytest_fly-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.22.5
+Root-Is-Purelib: true
+Tag: py3-none-any

pytest_fly-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+fly = pytest_fly.plugin

pytest_fly-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) 2024 James Abel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.