pytest-json-ctrf 0.1.0__py3-none-any.whl

plugin/BaseMetadataReport.py

@@ -0,0 +1,34 @@
+from pytest import Config, hookimpl, Item, CallInfo, CollectReport, Function, TestReport, fixture, FixtureRequest
+from .Report import Report
+from collections import OrderedDict
+
+metadata = '_ctrf_metadata'
+
+
+class BaseMetadataReport:
+    '''This class needed only to get metadata
+    from test nodes from different workers and
+    transfer it the report objects'''
+
+    def __init__(self):
+        self._report = Report()
+
+    def pytest_configure(self, config: Config):
+        if not hasattr(config, '_ctrf'):
+            setattr(config, '_ctrf', self)
+
+        if config.option.ctrf and not self._report:
+            self._report = Report()
+
+    @hookimpl(hookwrapper=True)
+    def pytest_runtest_protocol(self, item: Item, nextitem: Item):
+        setattr(item, metadata, {})
+        yield
+        delattr(item, metadata)
+
+    @hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(self, item, call):
+        report = (yield).get_result()
+        meta = getattr(item, metadata)
+        if meta:
+            setattr(report, metadata, meta)

plugin/CommonTestReportPlugin.py

@@ -0,0 +1,26 @@
+from pytest import Config, hookimpl, Item, CallInfo, CollectReport, Function, TestReport, fixture, FixtureRequest
+from .Report import Report
+from collections import OrderedDict
+from .BaseMetadataReport import metadata, BaseMetadataReport
+
+
+class CTRF(BaseMetadataReport):
+    def __init__(self):
+        super().__init__()
+        self._json_tests = OrderedDict()
+
+    @hookimpl(tryfirst=True)
+    def pytest_sessionstart(self, session):
+        if self._report:
+            self._report.start()
+
+    @hookimpl(tryfirst=True)
+    def pytest_sessionfinish(self, session):
+        if self._report:
+            self._report.stop()
+            self._report.save(session.config.option.ctrf)
+
+    def pytest_runtest_logreport(self, report: TestReport):
+        if not hasattr(report, metadata):
+            setattr(report, metadata, {})
+        self._report.collect(report)

plugin/Report.py

@@ -0,0 +1,78 @@
+import os
+import pytest
+import time
+import json
+from pytest import Item, CallInfo, TestReport, Mark
+from collections import OrderedDict
+from .TestObject import TestObject, TestStatus
+
+
+class Report:
+    def __init__(self):
+        self.test_items = OrderedDict()
+        self.prepared_tests = dict()
+        self.start_time = None
+        self.stop_time = None
+
+    def start(self) -> None:
+        self.start_time = time.time()
+
+    def stop(self) -> None:
+        self.stop_time = time.time()
+
+    @staticmethod
+    def _get_tool() -> dict:
+        return {
+            "name": "pytest",
+            "version": str(pytest.__version__)
+        }
+
+    def _get_summary(self) -> dict:
+        return {
+            'tests': len(self.prepared_tests),
+            'passed': len([test for test in self.prepared_tests.values() if test.status == TestStatus.PASSED]),
+            'failed': len([test for test in self.prepared_tests.values() if test.status == TestStatus.FAILED]),
+            'skipped': len([test for test in self.prepared_tests.values() if test.status == TestStatus.SKIPPED]),
+            'pending': len([test for test in self.prepared_tests.values() if test.status == TestStatus.PENDING]),
+            'other': 0,
+            'start': self.start_time,
+            'stop': self.stop_time
+        }
+
+    def collect(self, report: TestReport) -> None:
+        if report.nodeid not in self.test_items.keys():
+            test = TestObject()
+        else:
+            test = self.test_items.get(report.nodeid)
+        test.update(report, getattr(report, 'worker_id', None))
+        self.test_items[report.nodeid] = test
+
+    def process_retries(self) -> None:
+        for test_id, test_data in self.test_items.items():
+            name = test_id.split('[')[0]
+            test = self.prepared_tests.get(name)
+            if test:
+                test.retries += 1
+            else:
+                self.prepared_tests[name] = test_data
+
+    def get_report(self) -> dict:
+        self.process_retries()
+        return {'results':
+            {
+                "tool": self._get_tool(),
+                "summary": self._get_summary(),
+                "tests": [test.serialize() for test in self.prepared_tests.values()]
+            }
+        }
+
+    def save(self, report_file: str) -> None:
+        dirname = os.path.dirname(report_file)
+        if dirname:
+            try:
+                os.makedirs(dirname)
+            except Exception as e:
+                print(e)
+                raise Exception(f"Can't create directory {dirname}")
+        with open(report_file, 'w') as file:
+            json.dump(self.get_report(), file, default=str, indent=4)

plugin/TestObject.py

@@ -0,0 +1,73 @@
+from enum import Enum
+from pytest import TestReport
+
+
+class TestStatus(Enum):
+    PASSED = "passed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+    PENDING = "pending"
+    OTHER = "other"
+
+
+class TestObject:
+    name: str = None
+    status: TestStatus = TestStatus.PENDING
+    raw_status: str = None
+    duration: float = 0
+    start: float = None
+    stop: float = None
+    retries: int = 0
+
+    worker_id: str = None
+    file_path: str = None
+    tags: list[str] = None
+    browser: str = None
+    trace: str = None
+
+    def set_status(self, report: TestReport) -> None:
+        if self.status in (TestStatus.SKIPPED, TestStatus.FAILED):
+            return
+        elif report.skipped:
+            self.status = TestStatus.SKIPPED
+        elif report.failed:
+            self.status = TestStatus.FAILED
+            self.raw_status = f"{report.when}_{report.outcome}"
+        elif report.passed:
+            self.status = TestStatus.PASSED
+        else:
+            self.status = TestStatus.OTHER
+
+    def update(self, report: TestReport, worker_id: str = None) -> None:
+        self.name = report.head_line.split('[')[0]
+        self.set_status(report)
+        self.duration += report.duration
+        if report.when == "setup" and not self.start:
+            self.start = report.start
+        if report.when == "teardown" and not self.stop:
+            self.stop = report.stop
+        self.file_path = report.location[0]
+        if report.longrepr and len(report.longreprtext) > 0:
+            self.trace = report.longreprtext
+        if hasattr(report, '_ctrf_metadata'):
+            self.tags = report._ctrf_metadata.get('tags')
+            self.browser = report._ctrf_metadata.get('browser')
+        self.worker_id = worker_id
+
+    def serialize(self) -> dict:
+        result = {
+            'name': self.name,
+            'status': self.status.value,
+            'raw_status': self.raw_status,
+            'duration': self.duration,
+            'start': self.start,
+            'stop': self.stop,
+            'retries': None if self.retries == 1 else self.retries,
+            'file_path': self.file_path,
+            'tags': self.tags,
+            'browser': self.browser,
+            'trace': self.trace
+        }
+        if self.worker_id:
+            result['extra'] = {'worker': self.worker_id}
+        return {key: value for key, value in result.items() if value not in [None, '', []]}

plugin/main.py

@@ -0,0 +1,41 @@
+from pytest import Config, Parser, FixtureRequest, fixture, Function
+from plugin.CommonTestReportPlugin import CTRF
+from plugin.BaseMetadataReport import BaseMetadataReport
+
+
+def pytest_addoption(parser: Parser):
+    parser.addoption('--ctrf',
+                     action='store',
+                     help='generate test report. Report file name is optional')
+
+
+def pytest_configure(config: Config):
+    if not config.option.ctrf:
+        return
+    if hasattr(config, 'workerinput'):
+        plugin = BaseMetadataReport()
+    else:
+        plugin = CTRF()
+    setattr(config, '_ctrf', plugin)
+    config.pluginmanager.register(plugin, name='ctrf')
+
+
+def pytest_unconfigure(config: Config):
+    ctrf = getattr(config, '_ctrf', None)
+    if ctrf is not None:
+        delattr(config, '_ctrf')
+    if config.pluginmanager.hasplugin('ctrf'):
+        config.pluginmanager.unregister(name='ctrf')
+
+
+@fixture(autouse=True)
+def ctrf_json_metadata(request: FixtureRequest):
+    if not request.config.option.ctrf:
+        return
+    tags = [mark.name for mark in request.node.iter_markers()]
+    request.node._ctrf_metadata.setdefault('tags', tags)
+    if hasattr(request.node, 'callspec'):
+        browser = request.node.callspec.params.get('browser_name')
+        if browser:
+            request.node._ctrf_metadata.setdefault('browser', browser)
+    return request.node._ctrf_metadata

pytest_json_ctrf-0.1.0.dist-info/METADATA

@@ -0,0 +1,93 @@
+Metadata-Version: 2.1
+Name: pytest-json-ctrf
+Version: 0.1.0
+Summary: Pytest plugin to generate json report in CTRF (Common Test Report Format)
+Author: Oleksii Ostapov
+Project-URL: Infopulse, https://infopulse.com/
+Project-URL: Homepage, https://infopulse.com/pytest-json-ctrf
+Classifier: Framework :: Pytest
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pytest >6.0.0
+
+# CTRF for pytest
+
+Pytest implementation of Common Test Report Format (CTRF) for test results.  
+Test report will be generated in JSON format.  
+Test report can be used to prettify the report in GitHub Actions with [github-actions-ctrf](https://github.com/ctrf-io/github-actions-ctrf).  
+Do not worry if report in GitHub does not appear immediately. It takes some time to process the report. 
+
+## Features
+- Generates JSON report
+- Tested to work correctly with and without [pytest-xdist](https://pypi.org/project/pytest-xdist/)
+- Tested to get browser name from [pytest-playwright](https://pypi.org/project/pytest-playwright/)
+- Parametrized tests are supported and reported as 1 test with multiple retries
+
+## Installation
+
+```bash
+pip install pytest-json-ctrf
+```
+
+## Usage
+
+generate report.json file in the root directory of the project. File path is mandatory
+
+```bash
+pytest --ctrf report.json
+```
+
+## Json exampe
+
+More info here: https://ctrf.io/docs/schema/examples
+
+```json
+{
+  "results": {
+    "tool": {
+      "name": "jest"
+    },
+    "summary": {
+      "tests": 3,
+      "passed": 1,
+      "failed": 1,
+      "pending": 0,
+      "skipped": 1,
+      "other": 0,
+      "start": 1706644023,
+      "stop": 1706644043
+    },
+    "tests": [
+      {
+        "name": "User should be able to login",
+        "status": "passed",
+        "duration": 1200
+      },
+      {
+        "name": "User profile information should be correct",
+        "status": "failed",
+        "duration": 800
+      },
+      {
+        "name": "User should be able to logout",
+        "status": "skipped",
+        "duration": 0
+      }
+    ]
+  }
+}
+```
+
+## Report Example
+![Example Image](./assets/report_example.png)
+
+## Credits
+
+- https://ctrf.io/ -> nice data format
+- https://github.com/numirias/pytest-json-report -> Source of inspiration and dealing with xdist sync
+- https://github.com/testomatio/pytestomatio -> Source of inspiration for creating pytest plugins
+- https://github.com/infopulse/Playwright-course-python -> The report will be used in the demo project as +1 report option
+

pytest_json_ctrf-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+plugin/BaseMetadataReport.py,sha256=YWO8oYC5JXKOu6mHz1I1MlhGMiQXFmgk1rwOuMY81s4,1050
+plugin/CommonTestReportPlugin.py,sha256=1u5SN8MYjnaLgLgcpsed-Pal2wcSNq0vd-e4SKxTz1c,868
+plugin/Report.py,sha256=rLtjA81S64k05hMxUcJhehikEWrRJqlT04KnJbtbwSs,2659
+plugin/TestObject.py,sha256=fHIGuWtdfYX3nUVSoOZTodx1yOppyr2pazucyI641II,2401
+plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plugin/main.py,sha256=3JR8ZKEGeWcQgx4PInZe0ZjFx8_qf_VvDJa98xccBxA,1378
+pytest_json_ctrf-0.1.0.dist-info/METADATA,sha256=N8Jbu1A8__CkPvN0y_eA6LmlZUwbj1DkSkJEai9aQjo,2617
+pytest_json_ctrf-0.1.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+pytest_json_ctrf-0.1.0.dist-info/entry_points.txt,sha256=pKOt_brpWtWRajUgLmRfxbti6kTylIjf7kjtBeoA1QM,38
+pytest_json_ctrf-0.1.0.dist-info/top_level.txt,sha256=Io_dflkI6h0vZSGOzZxx4e76CDTSANVfu4v4tVY6zsA,7
+pytest_json_ctrf-0.1.0.dist-info/RECORD,,

pytest_json_ctrf-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pytest_json_ctrf-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+pytestomatio = plugin.main

pytest_json_ctrf-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+plugin