mypy-dev 1.9.0.dev0__py3-none-any.whl

mypy/modulefinder.py

@@ -0,0 +1,875 @@
+"""Low-level infrastructure to find modules.
+
+This builds on fscache.py; find_sources.py builds on top of this.
+"""
+
+from __future__ import annotations
+
+import ast
+import collections
+import functools
+import os
+import re
+import subprocess
+import sys
+from enum import Enum, unique
+from typing import Dict, Final, List, NamedTuple, Optional, Tuple, Union
+from typing_extensions import TypeAlias as _TypeAlias
+
+from mypy import pyinfo
+from mypy.errors import CompileError
+from mypy.fscache import FileSystemCache
+from mypy.nodes import MypyFile
+from mypy.options import Options
+from mypy.stubinfo import approved_stub_package_exists
+
+
+# Paths to be searched in find_module().
+class SearchPaths(NamedTuple):
+    python_path: tuple[str, ...]  # where user code is found
+    mypy_path: tuple[str, ...]  # from $MYPYPATH or config variable
+    package_path: tuple[str, ...]  # from get_site_packages_dirs()
+    typeshed_path: tuple[str, ...]  # paths in typeshed
+
+
+# Package dirs are a two-tuple of path to search and whether to verify the module
+OnePackageDir = Tuple[str, bool]
+PackageDirs = List[OnePackageDir]
+
+# Minimum and maximum Python versions for modules in stdlib as (major, minor)
+StdlibVersions: _TypeAlias = Dict[str, Tuple[Tuple[int, int], Optional[Tuple[int, int]]]]
+
+PYTHON_EXTENSIONS: Final = [".pyi", ".py"]
+
+
+# TODO: Consider adding more reasons here?
+# E.g. if we deduce a module would likely be found if the user were
+# to set the --namespace-packages flag.
+@unique
+class ModuleNotFoundReason(Enum):
+    # The module was not found: we found neither stubs nor a plausible code
+    # implementation (with or without a py.typed file).
+    NOT_FOUND = 0
+
+    # The implementation for this module plausibly exists (e.g. we
+    # found a matching folder or *.py file), but either the parent package
+    # did not contain a py.typed file or we were unable to find a
+    # corresponding *-stubs package.
+    FOUND_WITHOUT_TYPE_HINTS = 1
+
+    # The module was not found in the current working directory, but
+    # was able to be found in the parent directory.
+    WRONG_WORKING_DIRECTORY = 2
+
+    # Stub PyPI package (typically types-pkgname) known to exist but not installed.
+    APPROVED_STUBS_NOT_INSTALLED = 3
+
+    def error_message_templates(self, daemon: bool) -> tuple[str, list[str]]:
+        doc_link = "See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports"
+        if self is ModuleNotFoundReason.NOT_FOUND:
+            msg = 'Cannot find implementation or library stub for module named "{module}"'
+            notes = [doc_link]
+        elif self is ModuleNotFoundReason.WRONG_WORKING_DIRECTORY:
+            msg = 'Cannot find implementation or library stub for module named "{module}"'
+            notes = [
+                "You may be running mypy in a subpackage, "
+                "mypy should be run on the package root"
+            ]
+        elif self is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS:
+            msg = (
+                'Skipping analyzing "{module}": module is installed, but missing library stubs '
+                "or py.typed marker"
+            )
+            notes = [doc_link]
+        elif self is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
+            msg = 'Library stubs not installed for "{module}"'
+            notes = ['Hint: "python3 -m pip install {stub_dist}"']
+            if not daemon:
+                notes.append(
+                    '(or run "mypy --install-types" to install all missing stub packages)'
+                )
+            notes.append(doc_link)
+        else:
+            assert False
+        return msg, notes
+
+
+# If we found the module, returns the path to the module as a str.
+# Otherwise, returns the reason why the module wasn't found.
+ModuleSearchResult = Union[str, ModuleNotFoundReason]
+
+
+class BuildSource:
+    """A single source file."""
+
+    def __init__(
+        self,
+        path: str | None,
+        module: str | None,
+        text: str | None = None,
+        base_dir: str | None = None,
+        followed: bool = False,
+    ) -> None:
+        self.path = path  # File where it's found (e.g. 'xxx/yyy/foo/bar.py')
+        self.module = module or "__main__"  # Module name (e.g. 'foo.bar')
+        self.text = text  # Source code, if initially supplied, else None
+        self.base_dir = base_dir  # Directory where the package is rooted (e.g. 'xxx/yyy')
+        self.followed = followed  # Was this found by following imports?
+
+    def __repr__(self) -> str:
+        return (
+            "BuildSource(path={!r}, module={!r}, has_text={}, base_dir={!r}, followed={})".format(
+                self.path, self.module, self.text is not None, self.base_dir, self.followed
+            )
+        )
+
+
+class BuildSourceSet:
+    """Helper to efficiently test a file's membership in a set of build sources."""
+
+    def __init__(self, sources: list[BuildSource]) -> None:
+        self.source_text_present = False
+        self.source_modules: dict[str, str] = {}
+        self.source_paths: set[str] = set()
+
+        for source in sources:
+            if source.text is not None:
+                self.source_text_present = True
+            if source.path:
+                self.source_paths.add(source.path)
+            if source.module:
+                self.source_modules[source.module] = source.path or ""
+
+    def is_source(self, file: MypyFile) -> bool:
+        return (
+            (file.path and file.path in self.source_paths)
+            or file._fullname in self.source_modules
+            or self.source_text_present
+        )
+
+
+class FindModuleCache:
+    """Module finder with integrated cache.
+
+    Module locations and some intermediate results are cached internally
+    and can be cleared with the clear() method.
+
+    All file system accesses are performed through a FileSystemCache,
+    which is not ever cleared by this class. If necessary it must be
+    cleared by client code.
+    """
+
+    def __init__(
+        self,
+        search_paths: SearchPaths,
+        fscache: FileSystemCache | None,
+        options: Options | None,
+        stdlib_py_versions: StdlibVersions | None = None,
+        source_set: BuildSourceSet | None = None,
+    ) -> None:
+        self.search_paths = search_paths
+        self.source_set = source_set
+        self.fscache = fscache or FileSystemCache()
+        # Cache for get_toplevel_possibilities:
+        # search_paths -> (toplevel_id -> list(package_dirs))
+        self.initial_components: dict[tuple[str, ...], dict[str, list[str]]] = {}
+        # Cache find_module: id -> result
+        self.results: dict[str, ModuleSearchResult] = {}
+        self.ns_ancestors: dict[str, str] = {}
+        self.options = options
+        custom_typeshed_dir = None
+        if options:
+            custom_typeshed_dir = options.custom_typeshed_dir
+        self.stdlib_py_versions = stdlib_py_versions or load_stdlib_py_versions(
+            custom_typeshed_dir
+        )
+
+    def clear(self) -> None:
+        self.results.clear()
+        self.initial_components.clear()
+        self.ns_ancestors.clear()
+
+    def find_module_via_source_set(self, id: str) -> ModuleSearchResult | None:
+        """Fast path to find modules by looking through the input sources
+
+        This is only used when --fast-module-lookup is passed on the command line."""
+        if not self.source_set:
+            return None
+
+        p = self.source_set.source_modules.get(id, None)
+        if p and self.fscache.isfile(p):
+            # We need to make sure we still have __init__.py all the way up
+            # otherwise we might have false positives compared to slow path
+            # in case of deletion of init files, which is covered by some tests.
+            # TODO: are there some combination of flags in which this check should be skipped?
+            d = os.path.dirname(p)
+            for _ in range(id.count(".")):
+                if not any(
+                    self.fscache.isfile(os.path.join(d, "__init__" + x)) for x in PYTHON_EXTENSIONS
+                ):
+                    return None
+                d = os.path.dirname(d)
+            return p
+
+        idx = id.rfind(".")
+        if idx != -1:
+            # When we're looking for foo.bar.baz and can't find a matching module
+            # in the source set, look up for a foo.bar module.
+            parent = self.find_module_via_source_set(id[:idx])
+            if parent is None or not isinstance(parent, str):
+                return None
+
+            basename, ext = os.path.splitext(parent)
+            if not any(parent.endswith("__init__" + x) for x in PYTHON_EXTENSIONS) and (
+                ext in PYTHON_EXTENSIONS and not self.fscache.isdir(basename)
+            ):
+                # If we do find such a *module* (and crucially, we don't want a package,
+                # hence the filtering out of __init__ files, and checking for the presence
+                # of a folder with a matching name), then we can be pretty confident that
+                # 'baz' will either be a top-level variable in foo.bar, or will not exist.
+                #
+                # Either way, spelunking in other search paths for another 'foo.bar.baz'
+                # module should be avoided because:
+                #  1. in the unlikely event that one were found, it's highly likely that
+                #     it would be unrelated to the source being typechecked and therefore
+                #     more likely to lead to erroneous results
+                #  2. as described in _find_module, in some cases the search itself could
+                #  potentially waste significant amounts of time
+                return ModuleNotFoundReason.NOT_FOUND
+        return None
+
+    def find_lib_path_dirs(self, id: str, lib_path: tuple[str, ...]) -> PackageDirs:
+        """Find which elements of a lib_path have the directory a module needs to exist.
+
+        This is run for the python_path, mypy_path, and typeshed_path search paths.
+        """
+        components = id.split(".")
+        dir_chain = os.sep.join(components[:-1])  # e.g., 'foo/bar'
+
+        dirs = []
+        for pathitem in self.get_toplevel_possibilities(lib_path, components[0]):
+            # e.g., '/usr/lib/python3.4/foo/bar'
+            dir = os.path.normpath(os.path.join(pathitem, dir_chain))
+            if self.fscache.isdir(dir):
+                dirs.append((dir, True))
+        return dirs
+
+    def get_toplevel_possibilities(self, lib_path: tuple[str, ...], id: str) -> list[str]:
+        """Find which elements of lib_path could contain a particular top-level module.
+
+        In practice, almost all modules can be routed to the correct entry in
+        lib_path by looking at just the first component of the module name.
+
+        We take advantage of this by enumerating the contents of all of the
+        directories on the lib_path and building a map of which entries in
+        the lib_path could contain each potential top-level module that appears.
+        """
+
+        if lib_path in self.initial_components:
+            return self.initial_components[lib_path].get(id, [])
+
+        # Enumerate all the files in the directories on lib_path and produce the map
+        components: dict[str, list[str]] = {}
+        for dir in lib_path:
+            try:
+                contents = self.fscache.listdir(dir)
+            except OSError:
+                contents = []
+            # False positives are fine for correctness here, since we will check
+            # precisely later, so we only look at the root of every filename without
+            # any concern for the exact details.
+            for name in contents:
+                name = os.path.splitext(name)[0]
+                components.setdefault(name, []).append(dir)
+
+        self.initial_components[lib_path] = components
+        return components.get(id, [])
+
+    def find_module(self, id: str, *, fast_path: bool = False) -> ModuleSearchResult:
+        """Return the path of the module source file or why it wasn't found.
+
+        If fast_path is True, prioritize performance over generating detailed
+        error descriptions.
+        """
+        if id not in self.results:
+            top_level = id.partition(".")[0]
+            use_typeshed = True
+            if id in self.stdlib_py_versions:
+                use_typeshed = self._typeshed_has_version(id)
+            elif top_level in self.stdlib_py_versions:
+                use_typeshed = self._typeshed_has_version(top_level)
+            self.results[id] = self._find_module(id, use_typeshed)
+            if (
+                not (fast_path or (self.options is not None and self.options.fast_module_lookup))
+                and self.results[id] is ModuleNotFoundReason.NOT_FOUND
+                and self._can_find_module_in_parent_dir(id)
+            ):
+                self.results[id] = ModuleNotFoundReason.WRONG_WORKING_DIRECTORY
+        return self.results[id]
+
+    def _typeshed_has_version(self, module: str) -> bool:
+        if not self.options:
+            return True
+        version = typeshed_py_version(self.options)
+        min_version, max_version = self.stdlib_py_versions[module]
+        return version >= min_version and (max_version is None or version <= max_version)
+
+    def _find_module_non_stub_helper(
+        self, components: list[str], pkg_dir: str
+    ) -> OnePackageDir | ModuleNotFoundReason:
+        plausible_match = False
+        dir_path = pkg_dir
+        for index, component in enumerate(components):
+            dir_path = os.path.join(dir_path, component)
+            if self.fscache.isfile(os.path.join(dir_path, "py.typed")):
+                return os.path.join(pkg_dir, *components[:-1]), index == 0
+            elif not plausible_match and (
+                self.fscache.isdir(dir_path) or self.fscache.isfile(dir_path + ".py")
+            ):
+                plausible_match = True
+            # If this is not a directory then we can't traverse further into it
+            if not self.fscache.isdir(dir_path):
+                break
+        for i in range(len(components), 0, -1):
+            if approved_stub_package_exists(".".join(components[:i])):
+                return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
+        if plausible_match:
+            return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
+        else:
+            return ModuleNotFoundReason.NOT_FOUND
+
+    def _update_ns_ancestors(self, components: list[str], match: tuple[str, bool]) -> None:
+        path, verify = match
+        for i in range(1, len(components)):
+            pkg_id = ".".join(components[:-i])
+            if pkg_id not in self.ns_ancestors and self.fscache.isdir(path):
+                self.ns_ancestors[pkg_id] = path
+            path = os.path.dirname(path)
+
+    def _can_find_module_in_parent_dir(self, id: str) -> bool:
+        """Test if a module can be found by checking the parent directories
+        of the current working directory.
+        """
+        working_dir = os.getcwd()
+        parent_search = FindModuleCache(
+            SearchPaths((), (), (), ()),
+            self.fscache,
+            self.options,
+            stdlib_py_versions=self.stdlib_py_versions,
+        )
+        while any(is_init_file(file) for file in os.listdir(working_dir)):
+            working_dir = os.path.dirname(working_dir)
+            parent_search.search_paths = SearchPaths((working_dir,), (), (), ())
+            if not isinstance(parent_search._find_module(id, False), ModuleNotFoundReason):
+                return True
+        return False
+
+    def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
+        fscache = self.fscache
+
+        # Fast path for any modules in the current source set.
+        # This is particularly important when there are a large number of search
+        # paths which share the first (few) component(s) due to the use of namespace
+        # packages, for instance:
+        # foo/
+        #    company/
+        #        __init__.py
+        #        foo/
+        # bar/
+        #    company/
+        #        __init__.py
+        #        bar/
+        # baz/
+        #    company/
+        #        __init__.py
+        #        baz/
+        #
+        # mypy gets [foo/company/foo, bar/company/bar, baz/company/baz, ...] as input
+        # and computes [foo, bar, baz, ...] as the module search path.
+        #
+        # This would result in O(n) search for every import of company.*, leading to
+        # O(n**2) behavior in load_graph as such imports are unsurprisingly present
+        # at least once, and usually many more times than that, in each and every file
+        # being parsed.
+        #
+        # Thankfully, such cases are efficiently handled by looking up the module path
+        # via BuildSourceSet.
+        p = (
+            self.find_module_via_source_set(id)
+            if (self.options is not None and self.options.fast_module_lookup)
+            else None
+        )
+        if p:
+            return p
+
+        # If we're looking for a module like 'foo.bar.baz', it's likely that most of the
+        # many elements of lib_path don't even have a subdirectory 'foo/bar'.  Discover
+        # that only once and cache it for when we look for modules like 'foo.bar.blah'
+        # that will require the same subdirectory.
+        components = id.split(".")
+        dir_chain = os.sep.join(components[:-1])  # e.g., 'foo/bar'
+
+        # We have two sets of folders so that we collect *all* stubs folders and
+        # put them in the front of the search path
+        third_party_inline_dirs: PackageDirs = []
+        third_party_stubs_dirs: PackageDirs = []
+        found_possible_third_party_missing_type_hints = False
+        need_installed_stubs = False
+        # Third-party stub/typed packages
+        for pkg_dir in self.search_paths.package_path:
+            stub_name = components[0] + "-stubs"
+            stub_dir = os.path.join(pkg_dir, stub_name)
+            if fscache.isdir(stub_dir):
+                stub_typed_file = os.path.join(stub_dir, "py.typed")
+                stub_components = [stub_name] + components[1:]
+                path = os.path.join(pkg_dir, *stub_components[:-1])
+                if fscache.isdir(path):
+                    if fscache.isfile(stub_typed_file):
+                        # Stub packages can have a py.typed file, which must include
+                        # 'partial\n' to make the package partial
+                        # Partial here means that mypy should look at the runtime
+                        # package if installed.
+                        if fscache.read(stub_typed_file).decode().strip() == "partial":
+                            runtime_path = os.path.join(pkg_dir, dir_chain)
+                            third_party_inline_dirs.append((runtime_path, True))
+                            # if the package is partial, we don't verify the module, as
+                            # the partial stub package may not have a __init__.pyi
+                            third_party_stubs_dirs.append((path, False))
+                        else:
+                            # handle the edge case where people put a py.typed file
+                            # in a stub package, but it isn't partial
+                            third_party_stubs_dirs.append((path, True))
+                    else:
+                        third_party_stubs_dirs.append((path, True))
+            non_stub_match = self._find_module_non_stub_helper(components, pkg_dir)
+            if isinstance(non_stub_match, ModuleNotFoundReason):
+                if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS:
+                    found_possible_third_party_missing_type_hints = True
+                elif non_stub_match is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
+                    need_installed_stubs = True
+            else:
+                third_party_inline_dirs.append(non_stub_match)
+                self._update_ns_ancestors(components, non_stub_match)
+        if self.options and self.options.use_builtins_fixtures:
+            # Everything should be in fixtures.
+            third_party_inline_dirs.clear()
+            third_party_stubs_dirs.clear()
+            found_possible_third_party_missing_type_hints = False
+        python_mypy_path = self.search_paths.mypy_path + self.search_paths.python_path
+        candidate_base_dirs = self.find_lib_path_dirs(id, python_mypy_path)
+        if use_typeshed:
+            # Search for stdlib stubs in typeshed before installed
+            # stubs to avoid picking up backports (dataclasses, for
+            # example) when the library is included in stdlib.
+            candidate_base_dirs += self.find_lib_path_dirs(id, self.search_paths.typeshed_path)
+        candidate_base_dirs += third_party_stubs_dirs + third_party_inline_dirs
+
+        # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now
+        # contains just the subdirectories 'foo/bar' that actually exist under the
+        # elements of lib_path.  This is probably much shorter than lib_path itself.
+        # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories.
+        seplast = os.sep + components[-1]  # so e.g. '/baz'
+        sepinit = os.sep + "__init__"
+        near_misses = []  # Collect near misses for namespace mode (see below).
+        for base_dir, verify in candidate_base_dirs:
+            base_path = base_dir + seplast  # so e.g. '/usr/lib/python3.4/foo/bar/baz'
+            has_init = False
+            dir_prefix = base_dir
+            for _ in range(len(components) - 1):
+                dir_prefix = os.path.dirname(dir_prefix)
+            # Prefer package over module, i.e. baz/__init__.py* over baz.py*.
+            for extension in PYTHON_EXTENSIONS:
+                path = base_path + sepinit + extension
+                path_stubs = base_path + "-stubs" + sepinit + extension
+                if fscache.isfile_case(path, dir_prefix):
+                    has_init = True
+                    if verify and not verify_module(fscache, id, path, dir_prefix):
+                        near_misses.append((path, dir_prefix))
+                        continue
+                    return path
+                elif fscache.isfile_case(path_stubs, dir_prefix):
+                    if verify and not verify_module(fscache, id, path_stubs, dir_prefix):
+                        near_misses.append((path_stubs, dir_prefix))
+                        continue
+                    return path_stubs
+
+            # In namespace mode, register a potential namespace package
+            if self.options and self.options.namespace_packages:
+                if (
+                    not has_init
+                    and fscache.exists_case(base_path, dir_prefix)
+                    and not fscache.isfile_case(base_path, dir_prefix)
+                ):
+                    near_misses.append((base_path, dir_prefix))
+
+            # No package, look for module.
+            for extension in PYTHON_EXTENSIONS:
+                path = base_path + extension
+                if fscache.isfile_case(path, dir_prefix):
+                    if verify and not verify_module(fscache, id, path, dir_prefix):
+                        near_misses.append((path, dir_prefix))
+                        continue
+                    return path
+
+        # In namespace mode, re-check those entries that had 'verify'.
+        # Assume search path entries xxx, yyy and zzz, and we're
+        # looking for foo.bar.baz.  Suppose near_misses has:
+        #
+        # - xxx/foo/bar/baz.py
+        # - yyy/foo/bar/baz/__init__.py
+        # - zzz/foo/bar/baz.pyi
+        #
+        # If any of the foo directories has __init__.py[i], it wins.
+        # Else, we look for foo/bar/__init__.py[i], etc.  If there are
+        # none, the first hit wins.  Note that this does not take into
+        # account whether the lowest-level module is a file (baz.py),
+        # a package (baz/__init__.py), or a stub file (baz.pyi) -- for
+        # these the first one encountered along the search path wins.
+        #
+        # The helper function highest_init_level() returns an int that
+        # indicates the highest level at which a __init__.py[i] file
+        # is found; if no __init__ was found it returns 0, if we find
+        # only foo/bar/__init__.py it returns 1, and if we have
+        # foo/__init__.py it returns 2 (regardless of what's in
+        # foo/bar).  It doesn't look higher than that.
+        if self.options and self.options.namespace_packages and near_misses:
+            levels = [
+                highest_init_level(fscache, id, path, dir_prefix)
+                for path, dir_prefix in near_misses
+            ]
+            index = levels.index(max(levels))
+            return near_misses[index][0]
+
+        # Finally, we may be asked to produce an ancestor for an
+        # installed package with a py.typed marker that is a
+        # subpackage of a namespace package.  We only fess up to these
+        # if we would otherwise return "not found".
+        ancestor = self.ns_ancestors.get(id)
+        if ancestor is not None:
+            return ancestor
+
+        if need_installed_stubs:
+            return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
+        elif found_possible_third_party_missing_type_hints:
+            return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
+        else:
+            return ModuleNotFoundReason.NOT_FOUND
+
+    def find_modules_recursive(self, module: str) -> list[BuildSource]:
+        module_path = self.find_module(module)
+        if isinstance(module_path, ModuleNotFoundReason):
+            return []
+        sources = [BuildSource(module_path, module, None)]
+
+        package_path = None
+        if is_init_file(module_path):
+            package_path = os.path.dirname(module_path)
+        elif self.fscache.isdir(module_path):
+            package_path = module_path
+        if package_path is None:
+            return sources
+
+        # This logic closely mirrors that in find_sources. One small but important difference is
+        # that we do not sort names with keyfunc. The recursive call to find_modules_recursive
+        # calls find_module, which will handle the preference between packages, pyi and py.
+        # Another difference is it doesn't handle nested search paths / package roots.
+
+        seen: set[str] = set()
+        names = sorted(self.fscache.listdir(package_path))
+        for name in names:
+            # Skip certain names altogether
+            if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."):
+                continue
+            subpath = os.path.join(package_path, name)
+
+            if self.options and matches_exclude(
+                subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2
+            ):
+                continue
+
+            if self.fscache.isdir(subpath):
+                # Only recurse into packages
+                if (self.options and self.options.namespace_packages) or (
+                    self.fscache.isfile(os.path.join(subpath, "__init__.py"))
+                    or self.fscache.isfile(os.path.join(subpath, "__init__.pyi"))
+                ):
+                    seen.add(name)
+                    sources.extend(self.find_modules_recursive(module + "." + name))
+            else:
+                stem, suffix = os.path.splitext(name)
+                if stem == "__init__":
+                    continue
+                if stem not in seen and "." not in stem and suffix in PYTHON_EXTENSIONS:
+                    # (If we sorted names by keyfunc) we could probably just make the BuildSource
+                    # ourselves, but this ensures compatibility with find_module / the cache
+                    seen.add(stem)
+                    sources.extend(self.find_modules_recursive(module + "." + stem))
+        return sources
+
+
+def matches_exclude(
+    subpath: str, excludes: list[str], fscache: FileSystemCache, verbose: bool
+) -> bool:
+    if not excludes:
+        return False
+    subpath_str = os.path.relpath(subpath).replace(os.sep, "/")
+    if fscache.isdir(subpath):
+        subpath_str += "/"
+    for exclude in excludes:
+        if re.search(exclude, subpath_str):
+            if verbose:
+                print(
+                    f"TRACE: Excluding {subpath_str} (matches pattern {exclude})", file=sys.stderr
+                )
+            return True
+    return False
+
+
+def is_init_file(path: str) -> bool:
+    return os.path.basename(path) in ("__init__.py", "__init__.pyi")
+
+
+def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool:
+    """Check that all packages containing id have a __init__ file."""
+    if is_init_file(path):
+        path = os.path.dirname(path)
+    for i in range(id.count(".")):
+        path = os.path.dirname(path)
+        if not any(
+            fscache.isfile_case(os.path.join(path, f"__init__{extension}"), prefix)
+            for extension in PYTHON_EXTENSIONS
+        ):
+            return False
+    return True
+
+
+def highest_init_level(fscache: FileSystemCache, id: str, path: str, prefix: str) -> int:
+    """Compute the highest level where an __init__ file is found."""
+    if is_init_file(path):
+        path = os.path.dirname(path)
+    level = 0
+    for i in range(id.count(".")):
+        path = os.path.dirname(path)
+        if any(
+            fscache.isfile_case(os.path.join(path, f"__init__{extension}"), prefix)
+            for extension in PYTHON_EXTENSIONS
+        ):
+            level = i + 1
+    return level
+
+
+def mypy_path() -> list[str]:
+    path_env = os.getenv("MYPYPATH")
+    if not path_env:
+        return []
+    return path_env.split(os.pathsep)
+
+
+def default_lib_path(
+    data_dir: str, pyversion: tuple[int, int], custom_typeshed_dir: str | None
+) -> list[str]:
+    """Return default standard library search paths."""
+    path: list[str] = []
+
+    if custom_typeshed_dir:
+        typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib")
+        mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions")
+        versions_file = os.path.join(typeshed_dir, "VERSIONS")
+        if not os.path.isdir(typeshed_dir) or not os.path.isfile(versions_file):
+            print(
+                "error: --custom-typeshed-dir does not point to a valid typeshed ({})".format(
+                    custom_typeshed_dir
+                )
+            )
+            sys.exit(2)
+    else:
+        auto = os.path.join(data_dir, "stubs-auto")
+        if os.path.isdir(auto):
+            data_dir = auto
+        typeshed_dir = os.path.join(data_dir, "typeshed", "stdlib")
+        mypy_extensions_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-extensions")
+    path.append(typeshed_dir)
+
+    # Get mypy-extensions stubs from typeshed, since we treat it as an
+    # "internal" library, similar to typing and typing-extensions.
+    path.append(mypy_extensions_dir)
+
+    # Add fallback path that can be used if we have a broken installation.
+    if sys.platform != "win32":
+        path.append("/usr/local/lib/mypy")
+    if not path:
+        print(
+            "Could not resolve typeshed subdirectories. Your mypy install is broken.\n"
+            "Python executable is located at {}.\nMypy located at {}".format(
+                sys.executable, data_dir
+            ),
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    return path
+
+
+@functools.lru_cache(maxsize=None)
+def get_search_dirs(python_executable: str | None) -> tuple[list[str], list[str]]:
+    """Find package directories for given python.
+
+    This runs a subprocess call, which generates a list of the directories in sys.path.
+    To avoid repeatedly calling a subprocess (which can be slow!) we
+    lru_cache the results.
+    """
+
+    if python_executable is None:
+        return ([], [])
+    elif python_executable == sys.executable:
+        # Use running Python's package dirs
+        sys_path, site_packages = pyinfo.getsearchdirs()
+    else:
+        # Use subprocess to get the package directory of given Python
+        # executable
+        env = {**dict(os.environ), "PYTHONSAFEPATH": "1"}
+        try:
+            sys_path, site_packages = ast.literal_eval(
+                subprocess.check_output(
+                    [python_executable, pyinfo.__file__, "getsearchdirs"],
+                    env=env,
+                    stderr=subprocess.PIPE,
+                ).decode()
+            )
+        except subprocess.CalledProcessError as err:
+            print(err.stderr)
+            print(err.stdout)
+            raise
+        except OSError as err:
+            reason = os.strerror(err.errno)
+            raise CompileError(
+                [f"mypy: Invalid python executable '{python_executable}': {reason}"]
+            ) from err
+    return sys_path, site_packages
+
+
+def compute_search_paths(
+    sources: list[BuildSource], options: Options, data_dir: str, alt_lib_path: str | None = None
+) -> SearchPaths:
+    """Compute the search paths as specified in PEP 561.
+
+    There are the following 4 members created:
+    - User code (from `sources`)
+    - MYPYPATH (set either via config or environment variable)
+    - installed package directories (which will later be split into stub-only and inline)
+    - typeshed
+    """
+    # Determine the default module search path.
+    lib_path = collections.deque(
+        default_lib_path(
+            data_dir, options.python_version, custom_typeshed_dir=options.custom_typeshed_dir
+        )
+    )
+
+    if options.use_builtins_fixtures:
+        # Use stub builtins (to speed up test cases and to make them easier to
+        # debug).  This is a test-only feature, so assume our files are laid out
+        # as in the source tree.
+        # We also need to allow overriding where to look for it. Argh.
+        root_dir = os.getenv("MYPY_TEST_PREFIX", None)
+        if not root_dir:
+            root_dir = os.path.dirname(os.path.dirname(__file__))
+        lib_path.appendleft(os.path.join(root_dir, "test-data", "unit", "lib-stub"))
+    # alt_lib_path is used by some tests to bypass the normal lib_path mechanics.
+    # If we don't have one, grab directories of source files.
+    python_path: list[str] = []
+    if not alt_lib_path:
+        for source in sources:
+            # Include directory of the program file in the module search path.
+            if source.base_dir:
+                dir = source.base_dir
+                if dir not in python_path:
+                    python_path.append(dir)
+
+        # Do this even if running as a file, for sanity (mainly because with
+        # multiple builds, there could be a mix of files/modules, so its easier
+        # to just define the semantics that we always add the current director
+        # to the lib_path
+        # TODO: Don't do this in some cases; for motivation see see
+        # https://github.com/python/mypy/issues/4195#issuecomment-341915031
+        if options.bazel:
+            dir = "."
+        else:
+            dir = os.getcwd()
+        if dir not in lib_path:
+            python_path.insert(0, dir)
+
+    # Start with a MYPYPATH environment variable at the front of the mypy_path, if defined.
+    mypypath = mypy_path()
+
+    # Add a config-defined mypy path.
+    mypypath.extend(options.mypy_path)
+
+    # If provided, insert the caller-supplied extra module path to the
+    # beginning (highest priority) of the search path.
+    if alt_lib_path:
+        mypypath.insert(0, alt_lib_path)
+
+    sys_path, site_packages = get_search_dirs(options.python_executable)
+    # We only use site packages for this check
+    for site in site_packages:
+        assert site not in lib_path
+        if (
+            site in mypypath
+            or any(p.startswith(site + os.path.sep) for p in mypypath)
+            or (os.path.altsep and any(p.startswith(site + os.path.altsep) for p in mypypath))
+        ):
+            print(f"{site} is in the MYPYPATH. Please remove it.", file=sys.stderr)
+            print(
+                "See https://mypy.readthedocs.io/en/stable/running_mypy.html"
+                "#how-mypy-handles-imports for more info",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+    return SearchPaths(
+        python_path=tuple(reversed(python_path)),
+        mypy_path=tuple(mypypath),
+        package_path=tuple(sys_path + site_packages),
+        typeshed_path=tuple(lib_path),
+    )
+
+
+def load_stdlib_py_versions(custom_typeshed_dir: str | None) -> StdlibVersions:
+    """Return dict with minimum and maximum Python versions of stdlib modules.
+
+    The contents look like
+    {..., 'secrets': ((3, 6), None), 'symbol': ((2, 7), (3, 9)), ...}
+
+    None means there is no maximum version.
+    """
+    typeshed_dir = custom_typeshed_dir or os.path.join(os.path.dirname(__file__), "typeshed")
+    stdlib_dir = os.path.join(typeshed_dir, "stdlib")
+    result = {}
+
+    versions_path = os.path.join(stdlib_dir, "VERSIONS")
+    assert os.path.isfile(versions_path), (custom_typeshed_dir, versions_path, __file__)
+    with open(versions_path) as f:
+        for line in f:
+            line = line.split("#")[0].strip()
+            if line == "":
+                continue
+            module, version_range = line.split(":")
+            versions = version_range.split("-")
+            min_version = parse_version(versions[0])
+            max_version = (
+                parse_version(versions[1]) if len(versions) >= 2 and versions[1].strip() else None
+            )
+            result[module] = min_version, max_version
+    return result
+
+
+def parse_version(version: str) -> tuple[int, int]:
+    major, minor = version.strip().split(".")
+    return int(major), int(minor)
+
+
+def typeshed_py_version(options: Options) -> tuple[int, int]:
+    """Return Python version used for checking whether module supports typeshed."""
+    # Typeshed no longer covers Python 3.x versions before 3.7, so 3.7 is
+    # the earliest we can support.
+    return max(options.python_version, (3, 7))