pytest-testmon 2.1.4__tar.gz → 2.3.0b0__tar.gz

{pytest_testmon-2.1.4/pytest_testmon.egg-info → pytest_testmon-2.3.0b0}/PKG-INFO

@@ -1,9 +1,11 @@
 Metadata-Version: 2.4
 Name: pytest-testmon
-Version: 2.1.4
+Version: 2.3.0b0
 Summary: selects tests affected by changed files and methods
 Author-email: Tibor Arpas <tibor@testmon.org>
 License: MIT
+Project-URL: Homepage, https://testmon.org
+Project-URL: Source, https://github.com/tarpas/pytest-testmon
 Keywords: testing,pytest,plugin
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -15,10 +17,10 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Utilities
 Classifier: Programming Language :: Python
 Classifier: Framework :: Pytest
-Requires-Python: >=3.8
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pytest<9,>=5
+Requires-Dist: pytest<10,>=5
 Requires-Dist: coverage<8,>=6
 Dynamic: license-file
 

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/pyproject.toml

@@ -7,11 +7,11 @@ name = "pytest-testmon"
 authors = [
     { name = "Tibor Arpas", email = "tibor@testmon.org" },
 ]
-dynamic =  ["version"]
+dynamic = ["version"]
 
 description = "selects tests affected by changed files and methods"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.11"
 keywords = ["testing", "pytest", "plugin"]
 license = { text = "MIT" }
 classifiers = [
@@ -27,38 +27,32 @@ classifiers = [
     "Framework :: Pytest",
 ]
 dependencies = [
-    "pytest>=5,<9",
+    "pytest>=5,<10",
     "coverage>=6,<8",
 ]
 
-# specify testmon and tmserver as packages of this project
+[project.urls]
+Homepage = "https://testmon.org"
+Source = "https://github.com/tarpas/pytest-testmon"
+
+[project.entry-points.pytest11]
+pytest-testmon = "testmon.pytest_testmon"
+
 [tool.setuptools]
-packages = [
-    "testmon",
-]
+packages = ["testmon"]
 
 [tool.setuptools.dynamic]
 version = {attr = "testmon.TESTMON_VERSION"}
 
-[tool.black]
-exclude = '''
-(
-  /(
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.hg
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | buck-out
-    | build
-    | dist
-  )/
-  | io.intellij/
-)
-'''
-
-
-[project.entry-points.pytest11]
-pytest-testmon = "testmon.pytest_testmon"
+# Development dependencies, managed by uv: `uv sync` then `uv run pytest` / `uv run tox`.
+[dependency-groups]
+dev = [
+    "pytest==8.3.5",
+    "pytest-cov",
+    "pytest-xdist",
+    "pytest-mock",
+    "ruff",
+    "tox",
+    "tox-gh-actions",
+    "pre-commit",
+]

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0/pytest_testmon.egg-info}/PKG-INFO

@@ -1,9 +1,11 @@
 Metadata-Version: 2.4
 Name: pytest-testmon
-Version: 2.1.4
+Version: 2.3.0b0
 Summary: selects tests affected by changed files and methods
 Author-email: Tibor Arpas <tibor@testmon.org>
 License: MIT
+Project-URL: Homepage, https://testmon.org
+Project-URL: Source, https://github.com/tarpas/pytest-testmon
 Keywords: testing,pytest,plugin
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -15,10 +17,10 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Utilities
 Classifier: Programming Language :: Python
 Classifier: Framework :: Pytest
-Requires-Python: >=3.8
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pytest<9,>=5
+Requires-Dist: pytest<10,>=5
 Requires-Dist: coverage<8,>=6
 Dynamic: license-file
 

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/pytest_testmon.egg-info/requires.txt

@@ -1,2 +1,2 @@
-pytest<9,>=5
+pytest<10,>=5
 coverage<8,>=6

pytest_testmon-2.3.0b0/testmon/__init__.py

@@ -0,0 +1,3 @@
+"""PYTEST_DONT_REWRITE"""
+
+TESTMON_VERSION = "2.3.0b0"

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/testmon/configure.py

@@ -75,10 +75,12 @@ def _get_noselect_reasons(options):
     if options["markexpr"]:
         return ["-m was used"]
 
-    if options["lf"]:
+    # `--lf` is registered by the cacheprovider plugin, absent with `-p no:cacheprovider`
+    if options.get("lf"):
         return ["--lf was used"]
 
-    if any(re.match(r"(.*)\.py::(.*)", opt) for opt in options["file_or_dir"] or []):
+    file_or_dir = options.get("file_or_dir") or []
+    if any(re.match(r"(.*)\.py::(.*)", opt) for opt in file_or_dir):
         return ["you selected tests manually"]
 
     return []

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/testmon/db.py

@@ -5,12 +5,10 @@ import sqlite3
 from collections import namedtuple
 from functools import lru_cache
 
-from testmon.process_code import blob_to_checksums, checksums_to_blob
-
 from testmon.common import TestExecutions
+from testmon.process_code import blob_to_checksums, checksums_to_blob
 
-
-DATA_VERSION = 13
+DATA_VERSION = 14
 
 ChangedFileData = namedtuple(
     "ChangedFileData", "filename name method_checksums id failed"
@@ -81,12 +79,14 @@ class DB:  # pylint: disable=too-many-public-methods
     def version_compatibility(self):
         return DATA_VERSION
 
+    def close(self):
+        self.con.close()
+
     def __enter__(self):
-        self.con = self.con.__enter__()
         return self
 
-    def __exit__(self, *args, **kwargs):
-        self.con.__exit__(*args, **kwargs)
+    def __exit__(self, *args):
+        self.close()
 
     def _test_execution_fk_column(self) -> str:
         return "environment_id"
@@ -100,9 +100,7 @@ class DB:  # pylint: disable=too-many-public-methods
                 "UPDATE file_fp SET mtime=?, fsha=? WHERE id = ?", new_mtimes
             )
 
-    def finish_execution(
-        self, exec_id, duration=None, select=True
-    ):  # pylint: disable=unused-argument
+    def finish_execution(self, exec_id, duration=None, select=True):  # pylint: disable=unused-argument
         self.update_saving_stats(exec_id, select)
         self.fetch_or_create_file_fp.cache_clear()
         with self.con as con:
@@ -193,9 +191,7 @@ class DB:  # pylint: disable=too-many-public-methods
         )
 
     @lru_cache(1000)
-    def fetch_or_create_file_fp(
-        self, filename, fsha, method_checksums
-    ):  # pylint: disable=R0801
+    def fetch_or_create_file_fp(self, filename, fsha, method_checksums):  # pylint: disable=R0801
         cursor = self.con.cursor()
         try:
             cursor.execute(
@@ -216,9 +212,9 @@ class DB:  # pylint: disable=too-many-public-methods
                 FROM
                     file_fp
                 WHERE
-                    filename = ? AND method_checksums = ?
+                    filename = ? AND fsha = ? AND method_checksums = ?
                 """,
-                (filename, method_checksums),
+                (filename, fsha, method_checksums),
             ).fetchone()
 
         return fingerprint_id
@@ -359,7 +355,7 @@ class DB:  # pylint: disable=too-many-public-methods
                 duration FLOAT,
                 failed BIT,
                 forced BIT,
-                FOREIGN KEY({self._test_execution_fk_column()}) REFERENCES {self._test_execution_fk_table()}(id));
+                FOREIGN KEY({self._test_execution_fk_column()}) REFERENCES {self._test_execution_fk_table()}(id) ON DELETE CASCADE);
                 CREATE INDEX test_execution_fk_name ON test_execution ({self._test_execution_fk_column()}, test_name);
             """
 
@@ -394,7 +390,7 @@ class DB:  # pylint: disable=too-many-public-methods
             CREATE TABLE test_execution_file_fp (
                 test_execution_id INTEGER,
                 fingerprint_id INTEGER,
-                FOREIGN KEY(test_execution_id) REFERENCES test_execution(id),
+                FOREIGN KEY(test_execution_id) REFERENCES test_execution(id) ON DELETE CASCADE,
                 FOREIGN KEY(fingerprint_id) REFERENCES file_fp(id)
             );
             CREATE INDEX test_execution_file_fp_both ON test_execution_file_fp (test_execution_id, fingerprint_id);
@@ -403,7 +399,7 @@ class DB:  # pylint: disable=too-many-public-methods
                 suite_execution_id INTEGER,
                 filename TEXT,
                 fsha text,
-                FOREIGN KEY(suite_execution_id) REFERENCES suite_execution(id)
+                FOREIGN KEY(suite_execution_id) REFERENCES suite_execution(id) ON DELETE CASCADE
                 );
                 CREATE UNIQUE INDEX sefch_suite_id_filename_sha ON suite_execution_file_fsha(suite_execution_id, filename, fsha);
             """
@@ -559,23 +555,24 @@ class DB:  # pylint: disable=too-many-public-methods
             return {"affected": method_misses, "failing": failing_tests}
 
     def delete_test_executions(self, test_names, exec_id):
-        self.con.executemany(
-            f"""
-            DELETE
-            FROM test_execution_file_fp
-            WHERE test_execution_id IN
-                (SELECT id FROM test_execution WHERE {self._test_execution_fk_column()} = ? AND test_name = ?)
-                """,
-            [(exec_id, test_name) for test_name in test_names],
-        )
-        self.con.executemany(
-            f"""
-            DELETE
-            FROM test_execution
-            WHERE {self._test_execution_fk_column()} = ?
-              AND test_name = ?""",
-            [(exec_id, test_name) for test_name in test_names],
-        )
+        with self.con as con:
+            con.executemany(
+                f"""
+                DELETE
+                FROM test_execution_file_fp
+                WHERE test_execution_id IN
+                    (SELECT id FROM test_execution WHERE {self._test_execution_fk_column()} = ? AND test_name = ?)
+                    """,
+                [(exec_id, test_name) for test_name in test_names],
+            )
+            con.executemany(
+                f"""
+                DELETE
+                FROM test_execution
+                WHERE {self._test_execution_fk_column()} = ?
+                  AND test_name = ?""",
+                [(exec_id, test_name) for test_name in test_names],
+            )
 
     def all_test_executions(self, exec_id):
         return {
@@ -659,7 +656,7 @@ class DB:  # pylint: disable=too-many-public-methods
                 ORDER BY id DESC
                 """,
                 (environment_name,),
-            ).fetchone()
+            ).fetchone()  # TODO to pick best environment of many with the same name, we would need to fetch all.
 
             if environment:
                 environment_id = environment["id"]
@@ -669,7 +666,7 @@ class DB:  # pylint: disable=too-many-public-methods
                 )
             else:
                 packages_changed = False
-            if not environment or packages_changed:
+            if not environment:
                 try:
                     cursor.execute(
                         """
@@ -694,6 +691,19 @@ class DB:  # pylint: disable=too-many-public-methods
                         (environment_name,),
                     ).fetchone()
                     environment_id = environment["id"]
+            elif packages_changed:
+                cursor.execute(
+                    """
+                    INSERT INTO environment (environment_name, system_packages, python_version)
+                    VALUES (?, ?, ?)
+                    """,
+                    (environment_name, system_packages, python_version),
+                )
+                new_environment_id = cursor.lastrowid
+                cursor.execute(
+                    "DELETE FROM environment WHERE id = ?", (environment_id,)
+                )
+                environment_id = new_environment_id
 
             return environment_id, packages_changed
 

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/testmon/process_code.py

@@ -86,14 +86,17 @@ class Block:
 
 @lru_cache(300)
 def bytes_to_string_and_fsha(byte_stream: bytes) -> Union[str, bytes]:
-    # Replace \f because of http://bugs.python.org/issue19035
-    byte_stream = byte_stream.replace(b"\f", b" ")
+    # The fsha must equal git's blob sha of the CRLF-normalized bytes, so that it
+    # matches the index entry returned by get_files_shas() and no read is needed
+    # for files git reports as unmodified. The header length is the byte length,
+    # and hashing happens before the \f replacement (git does not touch \f).
     byte_stream = byte_stream.replace(b"\r\n", b"\n")
-    byte_string = byte_stream.decode(source_encoding(byte_stream), "replace")
-    git_header = b"blob %u\0" % len(byte_string)
     hsh = hashlib.sha1()
-    hsh.update(git_header)
+    hsh.update(b"blob %u\0" % len(byte_stream))
     hsh.update(byte_stream)
+    # Replace \f because of http://bugs.python.org/issue19035
+    byte_stream = byte_stream.replace(b"\f", b" ")
+    byte_string = byte_stream.decode(source_encoding(byte_stream), "replace")
     if byte_string and byte_string[-1] != "\n":
         byte_string += "\n"
     return byte_string, hsh.hexdigest()
@@ -235,13 +238,18 @@ def read_source_sha(filename: str):
 def noncached_get_files_shas(directory):
     all_shas = {}
     try:
+        # Run inside `directory` so the printed paths are relative to it (they
+        # are the keys testmon looks up); a pathspec argument would print paths
+        # relative to the current working directory instead.
         result = run(
-            ["git", "ls-files", "--stage", "-m", directory],
+            ["git", "ls-files", "--stage", "-m"],
+            cwd=directory or None,
             capture_output=True,
-            universal_newlines=True,
+            text=True,
+            encoding="utf-8",
             check=True,
         )
-    except (FileNotFoundError, CalledProcessError):
+    except (FileNotFoundError, NotADirectoryError, CalledProcessError):
         return all_shas
 
     modified_files = set()

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/testmon/pytest_testmon.py

@@ -2,6 +2,7 @@
 """
 Main module of testmon pytest plugin.
 """
+
 import time
 import xmlrpc.client
 import os
@@ -105,13 +106,18 @@ def pytest_addoption(parser):
             " code serving different endpoints or Django settings."
         ),
     )
-
+    group.addoption(
+        "--testmon-singleprocess",
+        action="store_false",
+        dest="testmon_singleprocess",
+        help="Don't track subprocesses",
+    )
     group.addoption(
         "--tmnet",
         action="store_true",
         dest="tmnet",
         help=(
-            "This is used for internal beta. Please don't use. You can go to https://www.testmon.net/ to register."
+            "Use tmnet cloud for storage instead of .testmondata file used by --testmon (see https://www.testmon.net)."
         ),
     )
 
@@ -132,6 +138,7 @@ def testmon_options(config):
         "testmon",
         "no-testmon",
         "environment_expression",
+        "testmon_singleprocess",
     ]:
         if config.getoption(label):
             result.append(label.replace("testmon_", ""))
@@ -165,24 +172,52 @@ def init_testmon_data(config: Config):
                     )
                 else:
                     tmnet_api_key = os.getenv("TMNET_API_KEY")
-            elif tmnet_api_key is None:
-                logger.warning(
-                    "TMNET_API_KEY not set.",
+
+            if not tmnet_api_key.strip():
+                raise ValueError(
+                    "TMNET_API_KEY is required when using --tmnet. "
+                    "Please set it in pytest.ini, pyproject.toml, or as an environment variable. "
                 )
+
             rpc_proxy = xmlrpc.client.ServerProxy(
                 url,
                 allow_none=True,
-                headers=[("x-api-key", tmnet_api_key)],
+                headers=[("x-api-key", tmnet_api_key.strip())],
             )
 
-    testmon_data = TestmonData(
-        rootdir=config.rootdir.strpath,
-        database=rpc_proxy,
-        environment=environment,
-        system_packages=system_packages,
-        readonly=get_running_as(config) == "worker",
-    )
-    testmon_data.determine_stable(bool(rpc_proxy))
+    # Check if we're a worker and have exec_id from controller
+    running_as = get_running_as(config)
+    exec_id = None
+    system_packages_change = None
+    files_of_interest = None
+
+    if running_as == "worker" and hasattr(config, "workerinput"):
+        # Use exec_id sent from controller
+        exec_id = config.workerinput.get("testmon_exec_id")
+        system_packages_change = config.workerinput.get(
+            "testmon_system_packages_change"
+        )
+        files_of_interest = config.workerinput.get("testmon_files_of_interest")
+
+    if running_as == "worker" and exec_id is not None:
+        # Initialize for xdist worker run
+        testmon_data: TestmonData = TestmonData.for_worker(
+            rootdir=str(config.rootpath),
+            exec_id=exec_id,
+            database=rpc_proxy,
+            system_packages_change=system_packages_change,
+            files_of_interest=files_of_interest,
+            environment=environment,
+        )
+    else:
+        # Initialize for local run (controller or single process)
+        testmon_data: TestmonData = TestmonData.for_local_run(
+            rootdir=str(config.rootpath),
+            database=rpc_proxy,
+            environment=environment,
+            system_packages=system_packages,
+        )
+    testmon_data.determine_stable()
     config.testmon_data = testmon_data
 
 
@@ -206,7 +241,7 @@ def register_plugins(config, should_select, should_collect, cov_plugin):
         config.pluginmanager.register(
             TestmonCollect(
                 TestmonCollector(
-                    config.rootdir.strpath,
+                    str(config.rootpath),
                     testmon_labels=testmon_options(config),
                     cov_plugin=cov_plugin,
                 ),
@@ -215,8 +250,8 @@ def register_plugins(config, should_select, should_collect, cov_plugin):
             ),
             "TestmonCollect",
         )
-        if config.pluginmanager.hasplugin("xdist"):
-            config.pluginmanager.register(TestmonXdistSync())
+    if (should_select or should_collect) and config.pluginmanager.hasplugin("xdist"):
+        config.pluginmanager.register(TestmonXdistSync(sync=should_collect))
 
 
 def pytest_configure(config):
@@ -238,6 +273,10 @@ def pytest_configure(config):
     )
     config.testmon_config: TmConf = tm_conf
     if tm_conf.select or tm_conf.collect:
+        if config.pluginmanager.has_plugin(
+            "TestmonSelect"
+        ) or config.pluginmanager.has_plugin("TestmonCollect"):
+            pytest.exit("You cannot use two instances of testmon plugin.")
         try:
             init_testmon_data(config)
             register_plugins(config, tm_conf.select, tm_conf.collect, cov_plugin)
@@ -321,7 +360,9 @@ def pytest_unconfigure(config):
 
 
 class TestmonCollect:
-    def __init__(self, testmon, testmon_data, running_as="single", cov_plugin=None):
+    def __init__(
+        self, testmon, testmon_data: TestmonData, running_as="single", cov_plugin=None
+    ):
         self.testmon_data: TestmonData = testmon_data
         self.testmon: TestmonCollector = testmon
         self._running_as = running_as
@@ -332,9 +373,7 @@ class TestmonCollect:
         self._sessionstarttime = time.time()
 
     @pytest.hookimpl(tryfirst=True, hookwrapper=True)
-    def pytest_pycollect_makeitem(
-        self, collector, name, obj
-    ):  # pylint: disable=unused-argument
+    def pytest_pycollect_makeitem(self, collector, name, obj):  # pylint: disable=unused-argument
         makeitem_result = yield
         items = makeitem_result.get_result() or []
         try:
@@ -345,9 +384,7 @@ class TestmonCollect:
             pass
 
     @pytest.hookimpl(tryfirst=True)
-    def pytest_collection_modifyitems(
-        self, session, config, items
-    ):  # pylint: disable=unused-argument
+    def pytest_collection_modifyitems(self, session, config, items):  # pylint: disable=unused-argument
         should_sync = not session.testsfailed and self._running_as in (
             "single",
             "controller",
@@ -356,9 +393,7 @@ class TestmonCollect:
             config.testmon_data.sync_db_fs_tests(retain=set(self.raw_test_names))
 
     @pytest.hookimpl(hookwrapper=True)
-    def pytest_runtest_protocol(
-        self, item, nextitem
-    ):  # pylint: disable=unused-argument
+    def pytest_runtest_protocol(self, item, nextitem):  # pylint: disable=unused-argument
         self.testmon.start_testmon(item.nodeid, nextitem.nodeid if nextitem else None)
         result = yield
         if result.excinfo and issubclass(result.excinfo[0], BaseException):
@@ -371,7 +406,9 @@ class TestmonCollect:
         if call.when == "teardown":
             report = result.get_result()
             report.nodes_files_lines = self.testmon.get_batch_coverage_data()
-            result.force_result(report)
+            result.force_result(
+                report
+            )  # under xdist, report is serialized on the worker and sent to the controller
 
     @pytest.hookimpl
     def pytest_runtest_logreport(self, report):
@@ -409,17 +446,44 @@ class TestmonCollect:
 
 
 class TestmonXdistSync:
-    def __init__(self):
+    def __init__(self, sync=True):
         self.await_nodes = 0
-
-    def pytest_testnodeready(self, node):  # pylint: disable=unused-argument
+        self.sync = sync
+
+    def pytest_configure_node(self, node):
+        """
+        Send exec_id and related data from controller to worker during xdist initialization.
+        This avoids each worker having to independently determine the environment and check
+        for package changes.
+
+        Note: This hook is called on the controller side for each worker node.
+        The node.config here is the controller's config, not the worker's config.
+        """
+        # Verify we're on the controller (not a worker)
+        # node.config in this hook is the controller's config
+        running_as = get_running_as(node.config)
+        if running_as != "controller":
+            return  # Safety check: only run on controller
+
+        # Count the node here, before it starts: counting it when it reports ready would let a
+        # fast worker's collection trigger sync_db_fs_tests while a slow one is still determining
+        # stability, which then sees the placeholder rows as stable tests and collects nothing.
         self.await_nodes += 1
 
-    def pytest_xdist_node_collection_finished(
-        self, node, ids
-    ):  # pylint: disable=invalid-name
+        # Only proceed if testmon_data has been initialized and workerinput exists
+        if hasattr(node.config, "testmon_data") and hasattr(node, "workerinput"):
+            testmon_data: TestmonData = node.config.testmon_data
+            node.workerinput["testmon_exec_id"] = testmon_data.exec_id
+            node.workerinput["testmon_system_packages_change"] = (
+                testmon_data.system_packages_change
+            )
+            node.workerinput["testmon_files_of_interest"] = (
+                testmon_data.files_of_interest
+            )
+
+    def pytest_xdist_node_collection_finished(self, node, ids):  # pylint: disable=invalid-name
         self.await_nodes += -1
-        if self.await_nodes == 0:
+        if self.await_nodes == 0 and self.sync:
             node.config.testmon_data.sync_db_fs_tests(retain=set(ids))
 
 
@@ -455,7 +519,7 @@ def format_time_saved(seconds):
 
 
 class TestmonSelect:
-    def __init__(self, config, testmon_data):
+    def __init__(self, config, testmon_data: TestmonData):
         self.testmon_data: TestmonData = testmon_data
         self.config = config
 
@@ -472,15 +536,13 @@ class TestmonSelect:
         self._interrupted = False
 
     def pytest_ignore_collect(self, collection_path: Path, config):
-        strpath = cached_relpath(str(collection_path), config.rootdir.strpath)
+        strpath = cached_relpath(str(collection_path), str(config.rootpath))
         if strpath in self.deselected_files and self.config.testmon_config.select:
             return True
         return None
 
     @pytest.hookimpl(trylast=True)
-    def pytest_collection_modifyitems(
-        self, session, config, items
-    ):  # pylint: disable=unused-argument
+    def pytest_collection_modifyitems(self, session, config, items):  # pylint: disable=unused-argument
         selected = []
         deselected = []
         for item in items:

{pytest_testmon-2.1.4 → pytest_testmon-2.3.0b0}/testmon/testmon_core.py

@@ -6,7 +6,7 @@ import sysconfig
 import textwrap
 from functools import lru_cache
 from collections import defaultdict
-from xmlrpc.client import Fault, ProtocolError
+from xmlrpc.client import Fault, ProtocolError, ServerProxy
 from socket import gaierror
 
 from typing import TypeVar
@@ -15,9 +15,9 @@ try:
     from pytest_cov.plugin import CovPlugin
 except ImportError:
     pass
-
 import pytest
 from coverage import Coverage, CoverageData
+from coverage.exceptions import CoverageException
 
 from testmon import db
 from testmon import TESTMON_VERSION as TM_CLIENT_VERSION
@@ -157,23 +157,14 @@ def collect_mhashes(source_tree, new_changed_file_data):
 class TestmonData:  # pylint: disable=too-many-instance-attributes
     __test__ = False
 
-    def __init__(  # pylint: disable=too-many-arguments
+    def __init__(
         self,
         rootdir,
         database=None,
-        environment=None,
-        system_packages=None,
-        python_version=None,
         readonly=False,
     ):
         self.rootdir = rootdir
-        self.environment = environment if environment else "default"
         self.source_tree = SourceTree(rootdir=self.rootdir)
-        if system_packages is None:
-            system_packages = get_system_packages()
-        system_packages = drop_patch_version(system_packages)
-        if not python_version:
-            python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
 
         if database:
             self.db = database  # pylint: disable=invalid-name
@@ -182,6 +173,48 @@ class TestmonData:  # pylint: disable=too-many-instance-attributes
                 os.path.join(self.rootdir, get_data_file_path()), readonly=readonly
             )  # pylint: disable=invalid-name
 
+        # Initialize instance variables (will be set by init methods)
+        self.environment = None
+        self.exec_id = None
+        self.system_packages_change = None
+        self.files_of_interest = None
+        self.all_files = {}
+        self.unstable_test_names = None
+        self.unstable_files = None
+        self.stable_test_names = None
+        self.stable_files = None
+        self.failing_tests = None
+
+    @classmethod
+    def for_local_run(
+        cls,
+        rootdir,
+        database=None,
+        environment=None,
+        system_packages=None,
+        python_version=None,
+    ):
+        instance = cls(rootdir, database=database, readonly=False)
+        instance._init_for_local_run(environment, system_packages, python_version)
+        return instance
+
+    def _init_for_local_run(
+        self,
+        environment=None,
+        system_packages=None,
+        python_version=None,
+    ):
+        """Internal method to initialize for local run."""
+        self.environment = environment if environment else "default"
+
+        if system_packages is None:
+            system_packages = get_system_packages()
+        system_packages = drop_patch_version(system_packages)
+
+        if not python_version:
+            python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+
+        # Initiate execution (controller or single process)
         try:
             result = self.db.initiate_execution(
                 self.environment,
@@ -201,32 +234,72 @@ class TestmonData:  # pylint: disable=too-many-instance-attributes
                 ),
                 exc,
             )
-            self.db = db.DB(
-                os.path.join(self.rootdir, get_data_file_path())
-            )  # pylint: disable=invalid-name
+            self.db = db.DB(os.path.join(self.rootdir, get_data_file_path()))  # pylint: disable=invalid-name
             result = self.db.initiate_execution(
                 self.environment, system_packages, python_version, {}
             )
-        self.exec_id = result["exec_id"]
 
+        self.exec_id = result["exec_id"]
         self.system_packages_change = result["packages_changed"]
         self.files_of_interest = result["filenames"]
 
-        self.all_files = {}
-        self.unstable_test_names = None
-        self.unstable_files = None
-        self.stable_test_names = None
-        self.stable_files = None
-        self.failing_tests = None
+    @classmethod
+    def for_worker(  # pylint: disable=too-many-arguments
+        cls,
+        rootdir,
+        exec_id,
+        database=None,
+        system_packages_change=None,
+        files_of_interest=None,
+        environment=None,
+    ):
+        instance = cls(rootdir, database=database, readonly=True)
+        instance._init_for_worker(
+            exec_id, system_packages_change, files_of_interest, environment
+        )
+        return instance
+
+    def _init_for_worker(
+        self,
+        exec_id,
+        system_packages_change=None,
+        files_of_interest=None,
+        environment=None,
+    ):
+        """Internal method to initialize for worker run."""
+        self.exec_id = exec_id
+        self.system_packages_change = system_packages_change
+        self.files_of_interest = files_of_interest
+        if environment:
+            self.environment = environment
+        elif not hasattr(self, "environment") or self.environment is None:
+            self.environment = "default"
 
     @property
     def new_db(self):
-        return self.db.file_created
+        # Only local DB instances have file_created attribute
+        if isinstance(self.db, db.DB):
+            return self.db.file_created
+        return False
 
     def close_connection(self):
-        pass
-        # TODO this is why there are "too many open files" on MacOS.
-        # we need the leave the connection open for tests
+        """Release the storage: the sqlite connection, or the transport of a tmnet proxy.
+
+        Idempotent. Tests that inspect the data after a run reopen it (see tests/lib4tests.py).
+        """
+        database, self.db = self.db, None
+        if database is None:
+            return
+        if isinstance(database, ServerProxy):
+            database("close")()  # attribute access would be a remote call
+        else:
+            database.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close_connection()
 
     @property
     def all_tests(self):
@@ -260,29 +333,27 @@ class TestmonData:  # pylint: disable=too-many-instance-attributes
     def sync_db_fs_tests(self, retain):
         collected = retain.union(set(self.stable_test_names))
         add = list(collected - set(self.all_tests))
-        with self.db:
-            test_execution_file_fps = {
-                test_name: {
-                    "deps": (
-                        {
-                            "filename": home_file(test_name),
-                            "method_checksums": methods_to_checksums(["0match"]),
-                            "mtime": None,
-                            "fsha": None,
-                        },
-                    )
-                }
-                for test_name in add
-                if is_python_file(home_file(test_name))
+        test_execution_file_fps = {
+            test_name: {
+                "deps": (
+                    {
+                        "filename": home_file(test_name),
+                        "method_checksums": methods_to_checksums(["0match"]),
+                        "mtime": None,
+                        "fsha": None,
+                    },
+                )
             }
-            if test_execution_file_fps:
-                self.save_test_execution_file_fps(test_execution_file_fps)
+            for test_name in add
+            if is_python_file(home_file(test_name))
+        }
+        if test_execution_file_fps:
+            self.save_test_execution_file_fps(test_execution_file_fps)
 
         to_delete = list(set(self.all_tests) - collected)
-        with self.db as database:
-            database.delete_test_executions(to_delete, self.exec_id)
+        self.db.delete_test_executions(to_delete, self.exec_id)
 
-    def determine_stable(self, assert_old=True):
+    def determine_stable(self):
         files_fshas = {}
         for filename in self.files_of_interest:
             module = self.source_tree.get_file(filename)
@@ -299,9 +370,6 @@ class TestmonData:  # pylint: disable=too-many-instance-attributes
         tests = self.db.determine_tests(self.exec_id, files_mhashes)
         affected_tests, self.failing_tests = tests["affected"], tests["failing"]
 
-        if assert_old:
-            self.assert_old_determin_stable(affected_tests)
-
         self.all_files = set(self.db.filenames(self.exec_id))
         self.unstable_test_names = set()
         self.unstable_files = set()
@@ -313,47 +381,6 @@ class TestmonData:  # pylint: disable=too-many-instance-attributes
         self.stable_test_names = set(self.all_tests) - self.unstable_test_names
         self.stable_files = set(self.all_files) - self.unstable_files
 
-    def assert_old_determin_stable(self, new_fingerprint_misses):
-        filenames_fingerprints = self.db.filenames_fingerprints(self.exec_id)
-
-        _, fsha_misses = split_filter(
-            self.source_tree, check_fsha, filenames_fingerprints
-        )  # check 2. fsha vs filesystem
-
-        # with the list of fingerprint_ids go to the database
-        # and fetch all the data needed for next step
-
-        changed_file_data = self.db.fetch_changed_file_data(
-            [fsha_miss["fingerprint_id"] for fsha_miss in (fsha_misses)],
-            self.exec_id,
-        )
-
-        # changed_file_data:
-        # [(filename, test_name, method_fshas, fingerprint_id, failed )]
-        # All the test_names in this list have a dependency on one
-        # or more changed files. And we also have the fingerprints
-        # of data content which they depend on. So it’s possible to
-        # filter out the node_ids where the content of the changed file
-        # still matches the fingerprint
-
-        _, fingerprint_misses = split_filter(
-            self.source_tree, check_fingerprint, changed_file_data
-        )
-
-        if {fingerprint_miss[1] for fingerprint_miss in fingerprint_misses} != set(
-            new_fingerprint_misses
-        ):
-            print("ERROR: old and new fingerprint misses differ.. printing old algo")
-            print(
-                "\n".join(
-                    sorted(
-                        {fingerprint_miss[1] for fingerprint_miss in fingerprint_misses}
-                    )
-                )
-            )
-            print("printing new algo")
-            print("\n".join(sorted(new_fingerprint_misses)))
-
     @property
     def avg_durations(self) -> dict:
         stats = defaultdict(lambda: {"test_execution": 0, "sum_duration": 0})
@@ -420,6 +447,22 @@ def cached_relpath(path, basepath):
     return os.path.relpath(path, basepath).replace(os.sep, "/")
 
 
+# coverage 7.10+ defaults to the sys.monitoring core on CPython 3.14+. That core
+# doesn't support dynamic contexts: sys.monitoring disables a line event once the
+# line has been recorded, so a line executed by several tests of one batch is
+# attributed to the first test only and the other tests lose the dependency.
+# testmon relies on contexts_by_lineno, so it needs the C tracer.
+COVERAGE_CORE = "ctrace"
+
+
+def force_context_capable_core(cov: Coverage):
+    try:
+        cov.set_option("run:core", COVERAGE_CORE)
+    except CoverageException:
+        # coverage < 7.10 has no core option and never defaults to sys.monitoring.
+        pass
+
+
 class TestmonCollector:
     coverage_stack: [Coverage] = []
 
@@ -499,9 +542,11 @@ class TestmonCollector:
                     "testmon doesn't support simultaneous run with pytest-cov when "
                     "branch coverage is on. Please disable branch coverage."
                 )
-
+        if subprocess:
+            self.setup_subprocess(params["include"])
         self.cov = Coverage(data_file=self.sub_cov_file, config_file=False, **params)
         self.cov._warn_no_data = False
+        force_context_capable_core(self.cov)
         if TestmonCollector.coverage_stack:
             TestmonCollector.coverage_stack[-1].stop()
 
@@ -600,11 +645,35 @@ class TestmonCollector:
             TestmonCollector.coverage_stack.pop()
         if self.sub_cov_file:
             os.remove(self.sub_cov_file + "_rc")
+            os.environ.pop("COVERAGE_CORE", None)
         os.environ.pop("COVERAGE_PROCESS_START", None)
         self.cov = None
         if TestmonCollector.coverage_stack:
             TestmonCollector.coverage_stack[-1].start()
 
+    def setup_subprocess(self, includes):
+        # This can be redone to use functionality from pytest-cov
+        if not os.path.exists(".tmontmp"):
+            os.makedirs(".tmontmp")
+        self.sub_cov_file = os.path.abspath(
+            ".tmontmp/.testmoncoverage" + str(random.randint(0, 1000000))
+        )
+        with open(self.sub_cov_file + "_rc", "w", encoding="utf-8") as subprocess_rc:
+            rc_content = textwrap.dedent(
+                """\
+                    [run]
+                    data_file = {}
+                    include = {}
+                    parallel=True
+                    """
+            ).format(
+                self.sub_cov_file,
+                "\n ".join(includes),
+            )
+            subprocess_rc.write(rc_content)
+        os.environ["COVERAGE_PROCESS_START"] = self.sub_cov_file + "_rc"
+        os.environ["COVERAGE_CORE"] = COVERAGE_CORE
+
 
 def eval_environment(environment, **kwargs):
     if not environment:

pytest_testmon-2.1.4/testmon/__init__.py

@@ -1,2 +0,0 @@
-"""PYTEST_DONT_REWRITE"""
-TESTMON_VERSION = "2.1.4"