pytest-testmon 2.1.1.dev0__tar.gz → 2.1.3__tar.gz

{pytest-testmon-2.1.1.dev0/pytest_testmon.egg-info → pytest_testmon-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytest-testmon
-Version: 2.1.1.dev0
+Version: 2.1.3
 Summary: selects tests affected by changed files and methods
 Author-email: Tibor Arpas <tibor@testmon.org>
 License: MIT
@@ -18,7 +18,7 @@ Classifier: Framework :: Pytest
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pytest<8,>=5
+Requires-Dist: pytest<9,>=5
 Requires-Dist: coverage<8,>=6
 
 <img src=https://user-images.githubusercontent.com/135344/219700265-0a9b152f-7285-4607-bbce-0c9aeddd520b.svg width=300>

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/pyproject.toml

@@ -27,10 +27,11 @@ classifiers = [
     "Framework :: Pytest",
 ]
 dependencies = [
-    "pytest>=5,<8",
+    "pytest>=5,<9",
     "coverage>=6,<8",
 ]
 
+# specify testmon and tmserver as packages of this project
 [tool.setuptools]
 packages = [
     "testmon",

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3/pytest_testmon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytest-testmon
-Version: 2.1.1.dev0
+Version: 2.1.3
 Summary: selects tests affected by changed files and methods
 Author-email: Tibor Arpas <tibor@testmon.org>
 License: MIT
@@ -18,7 +18,7 @@ Classifier: Framework :: Pytest
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pytest<8,>=5
+Requires-Dist: pytest<9,>=5
 Requires-Dist: coverage<8,>=6
 
 <img src=https://user-images.githubusercontent.com/135344/219700265-0a9b152f-7285-4607-bbce-0c9aeddd520b.svg width=300>

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/pytest_testmon.egg-info/requires.txt

@@ -1,2 +1,2 @@
-pytest<8,>=5
+pytest<9,>=5
 coverage<8,>=6

pytest_testmon-2.1.3/testmon/__init__.py

@@ -0,0 +1,2 @@
+"""PYTEST_DONT_REWRITE"""
+TESTMON_VERSION = "2.1.3"

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/common.py

@@ -3,6 +3,7 @@ import os
 import re
 
 try:
+    # Python >= 3.8
     import importlib.metadata
 
     def get_system_packages_raw():
@@ -12,34 +13,75 @@ try:
         )
 
 except ImportError:
+    # Python < 3.7
     import pkg_resources
 
     def get_system_packages_raw():
-        return ((pkg.project_name, pkg.version) for pkg in pkg_resources.working_set)
+        return (
+            (pkg.project_name, pkg.version)
+            for pkg in pkg_resources.working_set  # pylint: disable=not-an-iterable
+        )
 
 
 from pathlib import Path
 
+from typing import TypedDict, List, Dict
+
+
+class FileFp(TypedDict):
+    filename: str
+    method_checksums: List[int] = None
+    mtime: float = None  # optimization helper, not really a part of the data structure fundamentally
+    fsha: int = None  # optimization helper, not really a part of the data structure fundamentally
+    fingerprint_id: int = None  # optimization helper,
+
+
+TestName = str
+
+TestFileFps = Dict[TestName, List[FileFp]]
+
+Duration = float
+Failed = bool
+
+
+class DepsNOutcomes(TypedDict):
+    deps: List[FileFp]
+    failed: Failed
+    duration: Duration
+    forced: bool = None
+
+
+TestExecutions = Dict[TestName, DepsNOutcomes]
+
 
 def dummy():
     pass
 
 
 def get_logger(name):
-    logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
-    return logging.getLogger(name)
+    formatter = logging.Formatter("%(levelname)s: %(message)s")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+
+    # Configure the logger
+    tm_logger = logging.getLogger(name)
+    tm_logger.setLevel(logging.INFO)
+    tm_logger.addHandler(handler)
+    return tm_logger
 
 
 logger = get_logger(__name__)
 
 
 def get_system_packages(ignore=None):
+    if not ignore:
+        ignore = set(("pytest-testmon", "pytest-testmon"))
     return ", ".join(
         sorted(
             {
                 f"{package} {version}"
                 for (package, version) in get_system_packages_raw()
-                if not ignore or not package in ignore
+                if not package in ignore
             }
         )
     )
@@ -47,16 +89,19 @@ def get_system_packages(ignore=None):
 
 def drop_patch_version(system_packages):
     return re.sub(
-        r"\b([\w_-]+\s\d+\.\d+)\.\w+\b",
+        r"\b([\w_-]+\s\d+\.\d+)\.\w+\b",  # extract (Package M.N).P / drop .patch
         r"\1",
         system_packages,
     )
 
 
-def git_path(start_path=None):
+#
+# .git utilities
+#
+def git_path(start_path=None):  # parent dirs only
     start_path = Path(start_path or os.getcwd()).resolve()
     current_path = start_path
-    while current_path != current_path.parent:
+    while current_path != current_path.parent:  # '/'.parent == '/'
         path = current_path / ".git"
         if path.exists() and path.is_dir():
             return str(path)
@@ -73,7 +118,7 @@ def git_current_branch(path=None):
         with open(git_head_file, "r", encoding="utf8") as head_file:
             head_content = head_file.read().strip()
         if head_content.startswith("ref:"):
-            return head_content.split("/")[-1]
+            return head_content.split("/")[-1]  # e.g. ref: refs/heads/master
     except FileNotFoundError:
         pass
     return None

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/configure.py

@@ -2,7 +2,7 @@ import sys
 import re
 
 try:
-    from coverage.tracer import CTracer as Tracer
+    from coverage.tracer import CTracer as Tracer  # pylint: disable=no-name-in-module
 except ImportError:
     from coverage.pytracer import PyTracer as Tracer
 

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/db.py

@@ -7,6 +7,8 @@ from functools import lru_cache
 
 from testmon.process_code import blob_to_checksums, checksums_to_blob
 
+from testmon.common import TestExecutions
+
 
 DATA_VERSION = 13
 
@@ -20,10 +22,12 @@ class TestmonDbException(Exception):
 
 
 def connect(datafile, readonly=False):
-    connection = sqlite3.connect(
+    return sqlite3.connect(
         f"file:{datafile}{'?mode=ro' if readonly else ''}", uri=True, timeout=60
     )
 
+
+def connection_options(connection):
     connection.execute("PRAGMA journal_mode = WAL")
     connection.execute("PRAGMA synchronous = OFF")
     connection.execute("PRAGMA foreign_keys = TRUE ")
@@ -32,45 +36,51 @@ def connect(datafile, readonly=False):
     return connection
 
 
-def check_fingerprint_db(files_methods_checksums, record):
-    file = record[0]
-    fingerprint = record[2]
-
-    if file in files_methods_checksums and files_methods_checksums[file]:
-        if set(fingerprint) - set(files_methods_checksums[file]):
+def check_fingerprint_db(
+    files_methods_checksums, file_name, fingerprint: ChangedFileData
+):  # filename name method_checksums id failed
+    if file_name in files_methods_checksums and files_methods_checksums[file_name]:
+        if set(fingerprint) - set(files_methods_checksums[file_name]):
             return False
         return True
     return False
 
 
-class DB:
+def check_data_version(connection, datafile, data_version):
+    stored_data_version = connection.execute("PRAGMA user_version").fetchone()[0]
+
+    if int(stored_data_version) == data_version:
+        return connection, False
+
+    connection.close()
+    os.remove(datafile)
+    connection = connect(datafile)
+    connection = connection_options(connection)
+    return connection, True
+
+
+class DB:  # pylint: disable=too-many-public-methods
     def __init__(self, datafile, readonly=False):
         self._readonly = readonly
-        new_db = not os.path.exists(datafile)
+        file_exists = os.path.exists(datafile)
 
         connection = connect(datafile, readonly)
-        self.con = connection
-        old_format = self._check_data_version(datafile)
+        connection, old_format = check_data_version(
+            connection, datafile, self.version_compatibility()
+        )
+        self.con = connection_options(connection)
 
-        if new_db or old_format:
+        if (not file_exists) or old_format:
             self.init_tables()
+            self.file_created = True
+        else:
+            self.file_created = False
 
         self.con.executescript(self._local_temp_tables_statement())
 
     def version_compatibility(self):
         return DATA_VERSION
 
-    def _check_data_version(self, datafile):
-        stored_data_version = self._fetch_data_version()
-
-        if int(stored_data_version) == self.version_compatibility():
-            return False
-
-        self.con.close()
-        os.remove(datafile)
-        self.con = connect(datafile)
-        return True
-
     def __enter__(self):
         self.con = self.con.__enter__()
         return self
@@ -78,10 +88,10 @@ class DB:
     def __exit__(self, *args, **kwargs):
         self.con.__exit__(*args, **kwargs)
 
-    def _test_execution_fk_column(self):
+    def _test_execution_fk_column(self) -> str:
         return "environment_id"
 
-    def _test_execution_fk_table(self):
+    def _test_execution_fk_table(self) -> str:
         return "environment"
 
     def update_mtimes(self, new_mtimes):
@@ -90,7 +100,9 @@ class DB:
                 "UPDATE file_fp SET mtime=?, fsha=? WHERE id = ?", new_mtimes
             )
 
-    def finish_execution(self, exec_id, duration=None, select=True):
+    def finish_execution(
+        self, exec_id, duration=None, select=True
+    ):  # pylint: disable=unused-argument
         self.update_saving_stats(exec_id, select)
         self.fetch_or_create_file_fp.cache_clear()
         with self.con as con:
@@ -181,7 +193,9 @@ class DB:
         )
 
     @lru_cache(1000)
-    def fetch_or_create_file_fp(self, filename, fsha, method_checksums):
+    def fetch_or_create_file_fp(
+        self, filename, fsha, method_checksums
+    ):  # pylint: disable=R0801
         cursor = self.con.cursor()
         try:
             cursor.execute(
@@ -194,7 +208,7 @@ class DB:
             )
 
             fingerprint_id = cursor.lastrowid
-        except sqlite3.IntegrityError:
+        except sqlite3.IntegrityError:  # rather fetching existing fingerprint
             fingerprint_id, *_ = cursor.execute(
                 """
                 SELECT
@@ -209,15 +223,15 @@ class DB:
 
         return fingerprint_id
 
-    def _insert_test_execution(
+    def _insert_test_execution(  # pylint: disable=too-many-arguments
         self,
         con,
-        exec_id,
-        test_name,
-        duration,
-        failed,
-        forced,
-    ):
+        exec_id: int,
+        test_name: "str",
+        duration: "float",
+        failed: "bool",
+        forced: bool,
+    ) -> int:
         cursor = con.cursor()
         cursor.execute(
             f"""
@@ -235,7 +249,7 @@ class DB:
         )
         return cursor.lastrowid
 
-    def insert_test_file_fps(self, tests_deps_n_outcomes, exec_id=None):
+    def insert_test_file_fps(self, tests_deps_n_outcomes: TestExecutions, exec_id=None):
         assert exec_id
         with self.con as con:
             cursor = con.cursor()
@@ -285,11 +299,6 @@ class DB:
     def insert_into_suite_files_fshas(self, con, exec_id, files_fshas):
         pass
 
-    def _fetch_data_version(self):
-        con = self.con
-
-        return con.execute("PRAGMA user_version").fetchone()[0]
-
     def write_attribute(self, attribute, data, exec_id=None):
         dataid = f"{exec_id}:{attribute}"
         with self.con as con:
@@ -305,7 +314,7 @@ class DB:
         )
         result = cursor.fetchone()
         if result:
-            return json.loads(result[0])
+            return json.loads(result[0])  # zlib.decompress(result[0]).decode('utf-8)'))
         return default
 
     def increment_attributes(self, attributes_to_increment, exec_id=None):
@@ -328,10 +337,10 @@ class DB:
                     ],
                 )
 
-    def _create_metadata_statement(self):
+    def _create_metadata_statement(self) -> str:
         return """CREATE TABLE metadata (dataid TEXT PRIMARY KEY, data TEXT);"""
 
-    def _create_environment_statement(self):
+    def _create_environment_statement(self) -> str:
         return """
                 CREATE TABLE environment (
                 id INTEGER PRIMARY KEY ASC,
@@ -341,7 +350,7 @@ class DB:
                 UNIQUE (environment_name, system_packages, python_version)
             );"""
 
-    def _create_test_execution_statement(self):
+    def _create_test_execution_statement(self) -> str:  # pylint: disable=invalid-name
         return f"""
                 CREATE TABLE test_execution (
                 id INTEGER PRIMARY KEY ASC,
@@ -354,10 +363,10 @@ class DB:
                 CREATE INDEX test_execution_fk_name ON test_execution ({self._test_execution_fk_column()}, test_name);
             """
 
-    def _create_temp_tables_statement(self):
+    def _create_temp_tables_statement(self) -> str:
         return ""
 
-    def _local_temp_tables_statement(self):
+    def _local_temp_tables_statement(self) -> str:
         return """
                 CREATE TEMPORARY TABLE changed_files_fshas (exec_id INTEGER, filename TEXT, fsha TEXT);
                 CREATE INDEX changed_files_fshas_mcall ON changed_files_fshas (exec_id, filename, fsha);
@@ -366,7 +375,7 @@ class DB:
                 CREATE INDEX changed_files_mhashes_eid ON changed_files_mhashes (exec_id);
         """
 
-    def _create_file_fp_statement(self):
+    def _create_file_fp_statement(self) -> str:
         return """
             CREATE TABLE file_fp
             (
@@ -378,9 +387,9 @@ class DB:
                 UNIQUE (filename, fsha, method_checksums)
             );"""
 
-    def _create_test_execution_ffp_statement(
+    def _create_test_execution_ffp_statement(  # pylint: disable=invalid-name
         self,
-    ):
+    ) -> str:
         return """
             CREATE TABLE test_execution_file_fp (
                 test_execution_id INTEGER,
@@ -413,7 +422,7 @@ class DB:
 
         connection.execute(f"PRAGMA user_version = {self.version_compatibility()}")
 
-    def fetch_changed_file_data(self, changed_fingerprints, exec_id):
+    def fetch_changed_file_data(self, changed_fingerprints, exec_id) -> []:
         in_clause_questionsmarks = ", ".join("?" * len(changed_fingerprints))
         result = []
         for row in self.con.execute(
@@ -450,7 +459,9 @@ class DB:
 
         return result
 
-    def fetch_unknown_files(self, files_fshas, exec_id):
+    def fetch_unknown_files(
+        self, files_fshas, exec_id
+    ) -> []:  # exec_id is environment_id in this module
         with self.con as con:
             con.execute("DELETE FROM changed_files_fshas WHERE exec_id = ?", (exec_id,))
             con.executemany(
@@ -525,10 +536,10 @@ class DB:
                     ]
                 )
 
-            new_method_misses = []
+            method_misses = []
             for result in results:
-                if not check_fingerprint_db(files_mhashes, result):
-                    new_method_misses.append(result[1])
+                if not check_fingerprint_db(files_mhashes, result[0], result[2]):
+                    method_misses.append(result[1])
 
             failing_tests = [
                 row["test_name"]
@@ -545,7 +556,7 @@ class DB:
                 )
             ]
 
-            return {"affected": new_method_misses, "failing": failing_tests}
+            return {"affected": method_misses, "failing": failing_tests}
 
     def delete_test_executions(self, test_names, exec_id):
         self.con.executemany(
@@ -597,6 +608,7 @@ class DB:
 
         return [row[0] for row in cursor]
 
+    # TODO unify with filenames? Restrict not to go into ancient history, but not miss combinations?
     def all_filenames(self):
         cursor = self.con.execute(
             """
@@ -636,6 +648,7 @@ class DB:
         self, environment_name, system_packages, python_version
     ):
         with self.con as con:
+            con.execute("BEGIN IMMEDIATE TRANSACTION")
             cursor = con.cursor()
             environment = cursor.execute(
                 """
@@ -683,13 +696,13 @@ class DB:
 
             return environment_id, packages_changed
 
-    def initiate_execution(
+    def initiate_execution(  # pylint: disable= R0913 W0613
         self,
-        environment_name,
-        system_packages,
-        python_version,
-        execution_metadata,
-    ):
+        environment_name: str,
+        system_packages: str,
+        python_version: str,
+        execution_metadata: dict,  # pylint: disable=unused-argument
+    ) -> [int, list]:  # exec_id  # changed_file_data  # future_string2
         exec_id, packages_changed = self.fetch_or_create_environment(
             environment_name, system_packages, python_version
         )

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/process_code.py

@@ -31,7 +31,7 @@ def debug_blob_to_code(blob):
     return blob.split(";\n")
 
 
-def methods_to_checksums(blocks):
+def methods_to_checksums(blocks) -> [int]:
     checksums = []
     for block in blocks:
         checksums.append(to_signed(zlib.crc32(block.encode("UTF-8"))))
@@ -39,7 +39,7 @@ def methods_to_checksums(blocks):
     return checksums
 
 
-def checksums_to_blob(checksums):
+def checksums_to_blob(checksums: [int]) -> sqlite3.Binary:
     blob = array(CHECKUMS_ARRAY_TYPE, checksums)
     data = blob.tobytes()
     return sqlite3.Binary(data)
@@ -59,6 +59,7 @@ INVERTED_GAP_MARKS_CHECKSUMS = {
 
 class Block:
     def __init__(self, start, end, code=0, name=""):
+        # assert start <= end
         self.start = start
         self.end = end
         self.name = name
@@ -84,7 +85,8 @@ class Block:
 
 
 @lru_cache(300)
-def bytes_to_string_and_fsha(byte_stream):
+def bytes_to_string_and_fsha(byte_stream: bytes) -> Union[str, bytes]:
+    # Replace \f because of http://bugs.python.org/issue19035
     byte_stream = byte_stream.replace(b"\f", b" ")
     byte_stream = byte_stream.replace(b"\r\n", b"\n")
     byte_string = byte_stream.decode(source_encoding(byte_stream), "replace")
@@ -107,7 +109,7 @@ def _next_lineno(nodes, i, end):
 
 
 class Module:
-    def __init__(
+    def __init__(  # pylint: disable=too-many-arguments
         self,
         source_code=None,
         mtime=None,
@@ -130,6 +132,16 @@ class Module:
         self.ext = ext
 
     def dump_and_block(self, node, end, name="unknown", into_block=False):
+        """Frame of this method is taken from ast.dump
+        Objective is to return a representation of python source code where
+        all of the bodies of functions are replaced with 'transformed_into_block'
+        string. The rest of the syntax tree is represented in the same way as
+        in ast.dump(tree, annotate_fields=False). Of course the bodies of functions
+        are not completely thrown away, they are transformed into Block() objects
+        and appended to self.blocks. More can be probably understood from
+        (at the time rather messy) test_process_code.py examples.
+        """
+
         if isinstance(node, ast.AST):
             class_name = node.__class__.__name__
             fields = []
@@ -183,6 +195,8 @@ class Module:
                     tree = ast.parse(self.source_code, filename="<unknown>")
                     self.dump_and_block(tree, len(lines), name="<module>")
                 except SyntaxError:
+                    # We can continue without blocks because no tests depending on this file will ever get executed,
+                    # so no node depending on this checksum and mtime will ever be written to db.
                     pass
             else:
                 self._blocks = [Block(1, len(lines), self.source_code)]
@@ -199,8 +213,8 @@ class Module:
         return methods_to_checksums([block.checksum for block in self.blocks])
 
 
-def read_source_sha(filename):
-    # source_bytes: Optional[bytes]
+def read_source_sha(filename: str):
+    source_bytes: Optional[bytes]
 
     try:
         with open(filename, "rb") as file:
@@ -242,7 +256,7 @@ def get_files_shas(directory):
     return noncached_get_files_shas(directory)
 
 
-def get_source_sha(directory, filename):
+def get_source_sha(directory: "str", filename: "str"):
     try:
         sha = get_files_shas(directory)[filename]
         return (None, sha)
@@ -256,7 +270,7 @@ def match_fingerprint_source(source_code, fingerprint, ext="py"):
     return match_fingerprint(module, fingerprint)
 
 
-def match_fingerprint(module, fingerprint):
+def match_fingerprint(module: Module, fingerprint):
     if set(fingerprint) - set(module.checksums):
         return False
     return True
@@ -267,8 +281,8 @@ def create_fingerprint_source(source_code, lines, ext="py"):
     return create_fingerprint(module, lines)
 
 
-def create_fingerprint(module, covered_lines):
-    blocks = module.blocks
+def create_fingerprint(module, covered_lines) -> [int]:
+    blocks: [Block] = module.blocks
     method_reprs = []
     line_index = 0
     sorted_lines = sorted(covered_lines)

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/pytest_testmon.py

@@ -1,3 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+Main module of testmon pytest plugin.
+"""
 import time
 import xmlrpc.client
 import os
@@ -5,6 +9,7 @@ import os
 from collections import defaultdict
 from datetime import date, timedelta
 
+from pathlib import Path
 import pytest
 
 from _pytest.config import ExitCode, Config
@@ -133,7 +138,7 @@ def testmon_options(config):
     return result
 
 
-def init_testmon_data(config):
+def init_testmon_data(config: Config):
     environment = config.getoption("environment_expression") or eval_environment(
         config.getini("environment_expression")
     )
@@ -217,7 +222,7 @@ def register_plugins(config, should_select, should_collect, cov_plugin):
 def pytest_configure(config):
     coverage_stack = None
     try:
-        from tmnet.testmon_core import (
+        from tmnet.testmon_core import (  # pylint: disable=import-outside-toplevel
             Testmon as UberTestmon,
         )
 
@@ -231,7 +236,7 @@ def pytest_configure(config):
     tm_conf = configure.header_collect_select(
         config, coverage_stack, cov_plugin=cov_plugin
     )
-    config.testmon_config = tm_conf
+    config.testmon_config: TmConf = tm_conf
     if tm_conf.select or tm_conf.collect:
         try:
             init_testmon_data(config)
@@ -277,7 +282,7 @@ def pytest_report_header(config):
         if show_survey_notification:
             tm_conf.message += (
                 "\nWe'd like to hear from testmon users! "
-                "🙏🙏 go to https://testmon.org/survey to leave feedback ✅❌"
+                "Please go to https://testmon.org/survey to leave feedback."
             )
     return tm_conf.message
 
@@ -296,7 +301,7 @@ def changed_message(
         if changed_files_msg == "" or len(changed_files_msg) > 100:
             changed_files_msg = str(len(config.testmon_data.unstable_files))
 
-        if changed_files_msg == "0" and len(stable_files) == 0 and not packages_change:
+        if config.testmon_data.new_db:
             message += "new DB, "
         else:
             message += (
@@ -317,8 +322,8 @@ def pytest_unconfigure(config):
 
 class TestmonCollect:
     def __init__(self, testmon, testmon_data, running_as="single", cov_plugin=None):
-        self.testmon_data = testmon_data
-        self.testmon = testmon
+        self.testmon_data: TestmonData = testmon_data
+        self.testmon: TestmonCollector = testmon
         self._running_as = running_as
 
         self.reports = defaultdict(lambda: {})
@@ -327,18 +332,22 @@ class TestmonCollect:
         self._sessionstarttime = time.time()
 
     @pytest.hookimpl(tryfirst=True, hookwrapper=True)
-    def pytest_pycollect_makeitem(self, collector, name, obj):
+    def pytest_pycollect_makeitem(
+        self, collector, name, obj
+    ):  # pylint: disable=unused-argument
         makeitem_result = yield
         items = makeitem_result.get_result() or []
         try:
             self.raw_test_names.extend(
                 [item.nodeid for item in items if isinstance(item, pytest.Item)]
             )
-        except TypeError:
+        except TypeError:  # 'Class' object is not iterable
             pass
 
     @pytest.hookimpl(tryfirst=True)
-    def pytest_collection_modifyitems(self, session, config, items):
+    def pytest_collection_modifyitems(
+        self, session, config, items
+    ):  # pylint: disable=unused-argument
         should_sync = not session.testsfailed and self._running_as in (
             "single",
             "controller",
@@ -347,14 +356,16 @@ class TestmonCollect:
             config.testmon_data.sync_db_fs_tests(retain=set(self.raw_test_names))
 
     @pytest.hookimpl(hookwrapper=True)
-    def pytest_runtest_protocol(self, item, nextitem):
+    def pytest_runtest_protocol(
+        self, item, nextitem
+    ):  # pylint: disable=unused-argument
         self.testmon.start_testmon(item.nodeid, nextitem.nodeid if nextitem else None)
         result = yield
         if result.excinfo and issubclass(result.excinfo[0], BaseException):
             self.testmon.discard_current()
 
     @pytest.hookimpl(hookwrapper=True)
-    def pytest_runtest_makereport(self, item, call):
+    def pytest_runtest_makereport(self, item, call):  # pylint: disable=unused-argument
         result = yield
 
         if call.when == "teardown":
@@ -377,7 +388,7 @@ class TestmonCollect:
                     test_executions_fingerprints
                 )
 
-    def pytest_keyboard_interrupt(self, excinfo):
+    def pytest_keyboard_interrupt(self, excinfo):  # pylint: disable=unused-argument
         if self._running_as == "single":
             nodes_files_lines = self.testmon.get_batch_coverage_data()
 
@@ -387,7 +398,7 @@ class TestmonCollect:
             self.testmon_data.save_test_execution_file_fps(test_executions_fingerprints)
             self.testmon.close()
 
-    def pytest_sessionfinish(self, session):
+    def pytest_sessionfinish(self, session):  # pylint: disable=unused-argument
         if self._running_as in ("single", "controller"):
             self.testmon_data.db.finish_execution(
                 self.testmon_data.exec_id,
@@ -401,10 +412,12 @@ class TestmonXdistSync:
     def __init__(self):
         self.await_nodes = 0
 
-    def pytest_testnodeready(self, node):
+    def pytest_testnodeready(self, node):  # pylint: disable=unused-argument
         self.await_nodes += 1
 
-    def pytest_xdist_node_collection_finished(self, node, ids):
+    def pytest_xdist_node_collection_finished(
+        self, node, ids
+    ):  # pylint: disable=invalid-name
         self.await_nodes += -1
         if self.await_nodes == 0:
             node.config.testmon_data.sync_db_fs_tests(retain=set(ids))
@@ -423,7 +436,7 @@ def get_failing(all_test_executions):
     return failing_files, failing_tests
 
 
-def sort_items_by_duration(items, avg_durations):
+def sort_items_by_duration(items, avg_durations) -> None:
     items.sort(key=lambda item: avg_durations[item.nodeid])
     items.sort(
         key=lambda item: avg_durations[get_test_execution_class_name(item.nodeid)]
@@ -443,7 +456,7 @@ def format_time_saved(seconds):
 
 class TestmonSelect:
     def __init__(self, config, testmon_data):
-        self.testmon_data = testmon_data
+        self.testmon_data: TestmonData = testmon_data
         self.config = config
 
         failing_files, failing_test_names = get_failing(testmon_data.all_tests)
@@ -458,14 +471,16 @@ class TestmonSelect:
         ]
         self._interrupted = False
 
-    def pytest_ignore_collect(self, path, config):
-        strpath = cached_relpath(path.strpath, config.rootdir.strpath)
+    def pytest_ignore_collect(self, collection_path: Path, config):
+        strpath = cached_relpath(str(collection_path), config.rootdir.strpath)
         if strpath in self.deselected_files and self.config.testmon_config.select:
             return True
         return None
 
     @pytest.hookimpl(trylast=True)
-    def pytest_collection_modifyitems(self, session, config, items):
+    def pytest_collection_modifyitems(
+        self, session, config, items
+    ):  # pylint: disable=unused-argument
         selected = []
         deselected = []
         for item in items:
@@ -539,10 +554,10 @@ class TestmonSelect:
         )
         terminal_reporter.write_line(msg)
 
-    def pytest_keyboard_interrupt(self, excinfo):
+    def pytest_keyboard_interrupt(self, excinfo):  # pylint: disable=unused-argument
         self._interrupted = True
 
 
-class FakeItemFromTestmon:
+class FakeItemFromTestmon:  # pylint: disable=too-few-public-methods
     def __init__(self, config):
         self.config = config

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/testmon_core.py

@@ -9,6 +9,13 @@ from collections import defaultdict
 from xmlrpc.client import Fault, ProtocolError
 from socket import gaierror
 
+from typing import TypeVar
+
+try:
+    from pytest_cov.plugin import CovPlugin
+except ImportError:
+    pass
+
 import pytest
 from coverage import Coverage, CoverageData
 
@@ -29,6 +36,9 @@ from testmon.process_code import (
     Module,
 )
 
+from testmon.common import DepsNOutcomes, TestExecutions
+
+T = TypeVar("T")
 
 TEST_BATCH_SIZE = 250
 
@@ -55,10 +65,21 @@ class TestmonException(Exception):
 
 
 class SourceTree:
+    """
+    - reads files from file_system and caches them in memory
+      (if the file was once read, let's use the on memory
+      copy for further operations. Also caches the mtime.)
+    - store rootdir and convert between relative and absolute paths
+    - implement the check if the file is changed based on mtime, fsha.
+    - implement the ckeck if the node is stable based on fingerprint
+    - mockability (unit tests without filesystem, check if the file is changed
+      based on mtime, fsha)
+    """
+
     def __init__(self, rootdir, packages=None):
         self.rootdir = rootdir
         self.packages = packages
-        self.cache = {}
+        self.cache: dict = {}
 
     def get_file(self, filename):
         if filename not in self.cache:
@@ -81,7 +102,7 @@ class SourceTree:
         return self.cache[filename]
 
 
-def check_mtime(file_system, record):
+def check_mtime(file_system: SourceTree, record):
     absfilename = os.path.join(file_system.rootdir, record["filename"])
 
     cache_module = file_system.cache.get(record["filename"], None)
@@ -99,7 +120,9 @@ def check_fsha(file_system, record):
     return record["fsha"] == fs_fsha
 
 
-def check_fingerprint(disk, record):
+def check_fingerprint(
+    disk, record: db.ChangedFileData
+):  # filename name method_fshas id failed
     file = record[0]
     fingerprint = record[2]
 
@@ -107,7 +130,7 @@ def check_fingerprint(disk, record):
     return module and match_fingerprint(module, fingerprint)
 
 
-def split_filter(disk, function, records):
+def split_filter(disk, function, records: [T]) -> ([T], [T]):
     first = []
     second = []
     for record in records:
@@ -131,10 +154,10 @@ def collect_mhashes(source_tree, new_changed_file_data):
     return files_mhashes
 
 
-class TestmonData:
+class TestmonData:  # pylint: disable=too-many-instance-attributes
     __test__ = False
 
-    def __init__(
+    def __init__(  # pylint: disable=too-many-arguments
         self,
         rootdir,
         database=None,
@@ -145,7 +168,7 @@ class TestmonData:
     ):
         self.rootdir = rootdir
         self.environment = environment if environment else "default"
-        self.source_tree = SourceTree(rootdir=rootdir)
+        self.source_tree = SourceTree(rootdir=self.rootdir)
         if system_packages is None:
             system_packages = get_system_packages()
         system_packages = drop_patch_version(system_packages)
@@ -153,11 +176,11 @@ class TestmonData:
             python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
 
         if database:
-            self.db = database
+            self.db = database  # pylint: disable=invalid-name
         else:
             self.db = db.DB(
                 os.path.join(self.rootdir, get_data_file_path()), readonly=readonly
-            )
+            )  # pylint: disable=invalid-name
 
         try:
             result = self.db.initiate_execution(
@@ -178,7 +201,9 @@ class TestmonData:
                 ),
                 exc,
             )
-            self.db = db.DB(os.path.join(self.rootdir, get_data_file_path()))
+            self.db = db.DB(
+                os.path.join(self.rootdir, get_data_file_path())
+            )  # pylint: disable=invalid-name
             result = self.db.initiate_execution(
                 self.environment, system_packages, python_version, {}
             )
@@ -194,17 +219,23 @@ class TestmonData:
         self.stable_files = None
         self.failing_tests = None
 
+    @property
+    def new_db(self):
+        return self.db.file_created
+
     def close_connection(self):
         pass
+        # TODO this is why there are "too many open files" on MacOS.
+        # we need the leave the connection open for tests
 
     @property
     def all_tests(self):
         return self.db.all_test_executions(self.exec_id)
 
-    def get_tests_fingerprints(self, nodes_files_lines, reports):
+    def get_tests_fingerprints(self, nodes_files_lines, reports) -> TestExecutions:
         test_executions_fingerprints = {}
         for context in nodes_files_lines:
-            deps_n_outcomes = {"deps": []}
+            deps_n_outcomes: DepsNOutcomes = {"deps": []}
 
             for filename, covered in nodes_files_lines[context].items():
                 if os.path.exists(os.path.join(self.rootdir, filename)):
@@ -258,8 +289,11 @@ class TestmonData:
             if module:
                 files_fshas[filename] = module.fs_fsha
 
+        # Compare the fshas from disk to the fshas in the database and get files
+        # where the fsha is not in database.
         new_changed_file_data = self.db.fetch_unknown_files(files_fshas, self.exec_id)
 
+        # Get the mhashes for the files from above
         files_mhashes = collect_mhashes(self.source_tree, new_changed_file_data)
 
         tests = self.db.determine_tests(self.exec_id, files_mhashes)
@@ -284,13 +318,24 @@ class TestmonData:
 
         _, fsha_misses = split_filter(
             self.source_tree, check_fsha, filenames_fingerprints
-        )
+        )  # check 2. fsha vs filesystem
+
+        # with the list of fingerprint_ids go to the database
+        # and fetch all the data needed for next step
 
         changed_file_data = self.db.fetch_changed_file_data(
             [fsha_miss["fingerprint_id"] for fsha_miss in (fsha_misses)],
             self.exec_id,
         )
 
+        # changed_file_data:
+        # [(filename, test_name, method_fshas, fingerprint_id, failed )]
+        # All the test_names in this list have a dependency on one
+        # or more changed files. And we also have the fingerprints
+        # of data content which they depend on. So it’s possible to
+        # filter out the node_ids where the content of the changed file
+        # still matches the fingerprint
+
         _, fingerprint_misses = split_filter(
             self.source_tree, check_fingerprint, changed_file_data
         )
@@ -310,13 +355,13 @@ class TestmonData:
             print("\n".join(sorted(new_fingerprint_misses)))
 
     @property
-    def avg_durations(self):
+    def avg_durations(self) -> dict:
         stats = defaultdict(lambda: {"test_execution": 0, "sum_duration": 0})
 
         for (
             test_execution_id,
             report,
-        ) in self.all_tests.items():
+        ) in self.all_tests.items():  # pylint: disable=no-member
             if report:
                 class_name = get_test_execution_class_name(test_execution_id)
                 module_name = get_test_execution_module_name(test_execution_id)
@@ -347,6 +392,7 @@ class TestmonData:
 
 
 def get_new_mtimes(filesystem, hits):
+    """hits: [(filename, _, _, fingerprint_id)]"""
     try:
         for hit in hits:
             module = filesystem.get_file(hit[0])
@@ -375,24 +421,26 @@ def cached_relpath(path, basepath):
 
 
 class TestmonCollector:
-    coverage_stack = []
+    coverage_stack: [Coverage] = []
 
-    def __init__(self, rootdir, testmon_labels=None, cov_plugin=None):
+    def __init__(
+        self, rootdir, testmon_labels=None, cov_plugin=None
+    ):  # TODO remove cov_plugin
         try:
-            from testmon.testmon_core import (
+            from testmon.testmon_core import (  # pylint: disable=import-outside-toplevel
                 Testmon as UberTestmon,
             )
 
-            TestmonCollector.coverage_stack = UberTestmon.coverage_stack
+            TestmonCollector.coverage_stack: [Coverage] = UberTestmon.coverage_stack
         except ImportError:
             pass
         if testmon_labels is None:
             testmon_labels = {"singleprocess"}
         self.rootdir = rootdir
         self.testmon_labels = testmon_labels
-        self.cov = None
+        self.cov: Coverage = None
         self.sub_cov_file = None
-        self.cov_plugin = cov_plugin
+        self.cov_plugin: CovPlugin = cov_plugin
         self._test_name = None
         self._next_test_name = None
         self.batched_test_names = set()
@@ -445,6 +493,7 @@ class TestmonCollector:
                 params["include"] = list(
                     set(cov.config.run_include + params["include"])
                 )
+            # params["omit"] = cov.config.run_omit
             if cov.config.branch:
                 raise TestmonException(
                     "testmon doesn't support simultaneous run with pytest-cov when "
@@ -515,7 +564,7 @@ class TestmonCollector:
         return nodes_files_lines
 
     def get_nodes_files_lines(self, dont_include):
-        cov_data = self.cov.get_data()
+        cov_data: CoverageData = self.cov.get_data()
         files = cov_data.measured_files()
         nodes_files_lines = {}
         files_lines = {}
@@ -569,7 +618,7 @@ def eval_environment(environment, **kwargs):
 
     try:
         return str(eval(environment, eval_globals))
-    except Exception as error:
+    except Exception as error:  # pylint: disable=broad-except
         return repr(error)
 
 

{pytest-testmon-2.1.1.dev0 → pytest_testmon-2.1.3}/testmon/tox_testmon.py

@@ -1,5 +1,3 @@
-import os
-
 from pathlib import Path
 import pluggy
 
@@ -21,7 +19,7 @@ def touch_stampfile(venv):
 
 
 def installed_testmon(venv):
-    return os.path.exists(venv.path.join(".testmon_installed"))
+    return Path(venv.path.join(".testmon_installed")).exists()
 
 
 @hookimpl

pytest-testmon-2.1.1.dev0/testmon/__init__.py

@@ -1,2 +0,0 @@
-"""PYTEST_DONT_REWRITE"""
-TESTMON_VERSION = "2.1.1.dev0"