mypy-dev 1.9.0.dev0__py3-none-any.whl

mypy/fscache.py

@@ -0,0 +1,309 @@
+"""Interface for accessing the file system with automatic caching.
+
+The idea is to cache the results of any file system state reads during
+a single transaction. This has two main benefits:
+
+* This avoids redundant syscalls, as we won't perform the same OS
+  operations multiple times.
+
+* This makes it easier to reason about concurrent FS updates, as different
+  operations targeting the same paths can't report different state during
+  a transaction.
+
+Note that this only deals with reading state, not writing.
+
+Properties maintained by the API:
+
+* The contents of the file are always from the same or later time compared
+  to the reported mtime of the file, even if mtime is queried after reading
+  a file.
+
+* Repeating an operation produces the same result as the first one during
+  a transaction.
+
+* Call flush() to start a new transaction (flush the caches).
+
+The API is a bit limited. It's easy to add new cached operations, however.
+You should perform all file system reads through the API to actually take
+advantage of the benefits.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+
+from mypy_extensions import mypyc_attr
+
+from mypy.util import hash_digest
+
+
+@mypyc_attr(allow_interpreted_subclasses=True)  # for tests
+class FileSystemCache:
+    def __init__(self) -> None:
+        # The package root is not flushed with the caches.
+        # It is set by set_package_root() below.
+        self.package_root: list[str] = []
+        self.flush()
+
+    def set_package_root(self, package_root: list[str]) -> None:
+        self.package_root = package_root
+
+    def flush(self) -> None:
+        """Start another transaction and empty all caches."""
+        self.stat_cache: dict[str, os.stat_result] = {}
+        self.stat_error_cache: dict[str, OSError] = {}
+        self.listdir_cache: dict[str, list[str]] = {}
+        self.listdir_error_cache: dict[str, OSError] = {}
+        self.isfile_case_cache: dict[str, bool] = {}
+        self.exists_case_cache: dict[str, bool] = {}
+        self.read_cache: dict[str, bytes] = {}
+        self.read_error_cache: dict[str, Exception] = {}
+        self.hash_cache: dict[str, str] = {}
+        self.fake_package_cache: set[str] = set()
+
+    def stat(self, path: str) -> os.stat_result:
+        if path in self.stat_cache:
+            return self.stat_cache[path]
+        if path in self.stat_error_cache:
+            raise copy_os_error(self.stat_error_cache[path])
+        try:
+            st = os.stat(path)
+        except OSError as err:
+            if self.init_under_package_root(path):
+                try:
+                    return self._fake_init(path)
+                except OSError:
+                    pass
+            # Take a copy to get rid of associated traceback and frame objects.
+            # Just assigning to __traceback__ doesn't free them.
+            self.stat_error_cache[path] = copy_os_error(err)
+            raise err
+        self.stat_cache[path] = st
+        return st
+
+    def init_under_package_root(self, path: str) -> bool:
+        """Is this path an __init__.py under a package root?
+
+        This is used to detect packages that don't contain __init__.py
+        files, which is needed to support Bazel.  The function should
+        only be called for non-existing files.
+
+        It will return True if it refers to a __init__.py file that
+        Bazel would create, so that at runtime Python would think the
+        directory containing it is a package.  For this to work you
+        must pass one or more package roots using the --package-root
+        flag.
+
+        As an exceptional case, any directory that is a package root
+        itself will not be considered to contain a __init__.py file.
+        This is different from the rules Bazel itself applies, but is
+        necessary for mypy to properly distinguish packages from other
+        directories.
+
+        See https://docs.bazel.build/versions/master/be/python.html,
+        where this behavior is described under legacy_create_init.
+        """
+        if not self.package_root:
+            return False
+        dirname, basename = os.path.split(path)
+        if basename != "__init__.py":
+            return False
+        if not os.path.basename(dirname).isidentifier():
+            # Can't put an __init__.py in a place that's not an identifier
+            return False
+        try:
+            st = self.stat(dirname)
+        except OSError:
+            return False
+        else:
+            if not stat.S_ISDIR(st.st_mode):
+                return False
+        ok = False
+        drive, path = os.path.splitdrive(path)  # Ignore Windows drive name
+        if os.path.isabs(path):
+            path = os.path.relpath(path)
+        path = os.path.normpath(path)
+        for root in self.package_root:
+            if path.startswith(root):
+                if path == root + basename:
+                    # A package root itself is never a package.
+                    ok = False
+                    break
+                else:
+                    ok = True
+        return ok
+
+    def _fake_init(self, path: str) -> os.stat_result:
+        """Prime the cache with a fake __init__.py file.
+
+        This makes code that looks for path believe an empty file by
+        that name exists.  Should only be called after
+        init_under_package_root() returns True.
+        """
+        dirname, basename = os.path.split(path)
+        assert basename == "__init__.py", path
+        assert not os.path.exists(path), path  # Not cached!
+        dirname = os.path.normpath(dirname)
+        st = self.stat(dirname)  # May raise OSError
+        # Get stat result as a list so we can modify it.
+        seq: list[float] = list(st)
+        seq[stat.ST_MODE] = stat.S_IFREG | 0o444
+        seq[stat.ST_INO] = 1
+        seq[stat.ST_NLINK] = 1
+        seq[stat.ST_SIZE] = 0
+        st = os.stat_result(seq)
+        self.stat_cache[path] = st
+        # Make listdir() and read() also pretend this file exists.
+        self.fake_package_cache.add(dirname)
+        return st
+
+    def listdir(self, path: str) -> list[str]:
+        path = os.path.normpath(path)
+        if path in self.listdir_cache:
+            res = self.listdir_cache[path]
+            # Check the fake cache.
+            if path in self.fake_package_cache and "__init__.py" not in res:
+                res.append("__init__.py")  # Updates the result as well as the cache
+            return res
+        if path in self.listdir_error_cache:
+            raise copy_os_error(self.listdir_error_cache[path])
+        try:
+            results = os.listdir(path)
+        except OSError as err:
+            # Like above, take a copy to reduce memory use.
+            self.listdir_error_cache[path] = copy_os_error(err)
+            raise err
+        self.listdir_cache[path] = results
+        # Check the fake cache.
+        if path in self.fake_package_cache and "__init__.py" not in results:
+            results.append("__init__.py")
+        return results
+
+    def isfile(self, path: str) -> bool:
+        try:
+            st = self.stat(path)
+        except OSError:
+            return False
+        return stat.S_ISREG(st.st_mode)
+
+    def isfile_case(self, path: str, prefix: str) -> bool:
+        """Return whether path exists and is a file.
+
+        On case-insensitive filesystems (like Mac or Windows) this returns
+        False if the case of path's last component does not exactly match
+        the case found in the filesystem.
+
+        We check also the case of other path components up to prefix.
+        For example, if path is 'user-stubs/pack/mod.pyi' and prefix is 'user-stubs',
+        we check that the case of 'pack' and 'mod.py' matches exactly, 'user-stubs' will be
+        case insensitive on case insensitive filesystems.
+
+        The caller must ensure that prefix is a valid file system prefix of path.
+        """
+        if not self.isfile(path):
+            # Fast path
+            return False
+        if path in self.isfile_case_cache:
+            return self.isfile_case_cache[path]
+        head, tail = os.path.split(path)
+        if not tail:
+            self.isfile_case_cache[path] = False
+            return False
+        try:
+            names = self.listdir(head)
+            # This allows one to check file name case sensitively in
+            # case-insensitive filesystems.
+            res = tail in names
+        except OSError:
+            res = False
+        if res:
+            # Also recursively check the other path components in case sensitive way.
+            res = self.exists_case(head, prefix)
+        self.isfile_case_cache[path] = res
+        return res
+
+    def exists_case(self, path: str, prefix: str) -> bool:
+        """Return whether path exists - checking path components in case sensitive
+        fashion, up to prefix.
+        """
+        if path in self.exists_case_cache:
+            return self.exists_case_cache[path]
+        head, tail = os.path.split(path)
+        if not head.startswith(prefix) or not tail:
+            # Only perform the check for paths under prefix.
+            self.exists_case_cache[path] = True
+            return True
+        try:
+            names = self.listdir(head)
+            # This allows one to check file name case sensitively in
+            # case-insensitive filesystems.
+            res = tail in names
+        except OSError:
+            res = False
+        if res:
+            # Also recursively check other path components.
+            res = self.exists_case(head, prefix)
+        self.exists_case_cache[path] = res
+        return res
+
+    def isdir(self, path: str) -> bool:
+        try:
+            st = self.stat(path)
+        except OSError:
+            return False
+        return stat.S_ISDIR(st.st_mode)
+
+    def exists(self, path: str) -> bool:
+        try:
+            self.stat(path)
+        except FileNotFoundError:
+            return False
+        return True
+
+    def read(self, path: str) -> bytes:
+        if path in self.read_cache:
+            return self.read_cache[path]
+        if path in self.read_error_cache:
+            raise self.read_error_cache[path]
+
+        # Need to stat first so that the contents of file are from no
+        # earlier instant than the mtime reported by self.stat().
+        self.stat(path)
+
+        dirname, basename = os.path.split(path)
+        dirname = os.path.normpath(dirname)
+        # Check the fake cache.
+        if basename == "__init__.py" and dirname in self.fake_package_cache:
+            data = b""
+        else:
+            try:
+                with open(path, "rb") as f:
+                    data = f.read()
+            except OSError as err:
+                self.read_error_cache[path] = err
+                raise
+
+        self.read_cache[path] = data
+        self.hash_cache[path] = hash_digest(data)
+        return data
+
+    def hash_digest(self, path: str) -> str:
+        if path not in self.hash_cache:
+            self.read(path)
+        return self.hash_cache[path]
+
+    def samefile(self, f1: str, f2: str) -> bool:
+        s1 = self.stat(f1)
+        s2 = self.stat(f2)
+        return os.path.samestat(s1, s2)
+
+
+def copy_os_error(e: OSError) -> OSError:
+    new = OSError(*e.args)
+    new.errno = e.errno
+    new.strerror = e.strerror
+    new.filename = e.filename
+    if e.filename2:
+        new.filename2 = e.filename2
+    return new

mypy/fswatcher.py

@@ -0,0 +1,106 @@
+"""Watch parts of the file system for changes."""
+
+from __future__ import annotations
+
+from typing import AbstractSet, Iterable, NamedTuple
+
+from mypy.fscache import FileSystemCache
+
+
+class FileData(NamedTuple):
+    st_mtime: float
+    st_size: int
+    hash: str
+
+
+class FileSystemWatcher:
+    """Watcher for file system changes among specific paths.
+
+    All file system access is performed using FileSystemCache. We
+    detect changed files by stat()ing them all and comparing hashes
+    of potentially changed files. If a file has both size and mtime
+    unmodified, the file is assumed to be unchanged.
+
+    An important goal of this class is to make it easier to eventually
+    use file system events to detect file changes.
+
+    Note: This class doesn't flush the file system cache. If you don't
+    manually flush it, changes won't be seen.
+    """
+
+    # TODO: Watching directories?
+    # TODO: Handle non-files
+
+    def __init__(self, fs: FileSystemCache) -> None:
+        self.fs = fs
+        self._paths: set[str] = set()
+        self._file_data: dict[str, FileData | None] = {}
+
+    def dump_file_data(self) -> dict[str, tuple[float, int, str]]:
+        return {k: v for k, v in self._file_data.items() if v is not None}
+
+    def set_file_data(self, path: str, data: FileData) -> None:
+        self._file_data[path] = data
+
+    def add_watched_paths(self, paths: Iterable[str]) -> None:
+        for path in paths:
+            if path not in self._paths:
+                # By storing None this path will get reported as changed by
+                # find_changed if it exists.
+                self._file_data[path] = None
+        self._paths |= set(paths)
+
+    def remove_watched_paths(self, paths: Iterable[str]) -> None:
+        for path in paths:
+            if path in self._file_data:
+                del self._file_data[path]
+        self._paths -= set(paths)
+
+    def _update(self, path: str) -> None:
+        st = self.fs.stat(path)
+        hash_digest = self.fs.hash_digest(path)
+        self._file_data[path] = FileData(st.st_mtime, st.st_size, hash_digest)
+
+    def _find_changed(self, paths: Iterable[str]) -> AbstractSet[str]:
+        changed = set()
+        for path in paths:
+            old = self._file_data[path]
+            try:
+                st = self.fs.stat(path)
+            except FileNotFoundError:
+                if old is not None:
+                    # File was deleted.
+                    changed.add(path)
+                    self._file_data[path] = None
+            else:
+                if old is None:
+                    # File is new.
+                    changed.add(path)
+                    self._update(path)
+                # Round mtimes down, to match the mtimes we write to meta files
+                elif st.st_size != old.st_size or int(st.st_mtime) != int(old.st_mtime):
+                    # Only look for changes if size or mtime has changed as an
+                    # optimization, since calculating hash is expensive.
+                    new_hash = self.fs.hash_digest(path)
+                    self._update(path)
+                    if st.st_size != old.st_size or new_hash != old.hash:
+                        # Changed file.
+                        changed.add(path)
+        return changed
+
+    def find_changed(self) -> AbstractSet[str]:
+        """Return paths that have changes since the last call, in the watched set."""
+        return self._find_changed(self._paths)
+
+    def update_changed(self, remove: list[str], update: list[str]) -> AbstractSet[str]:
+        """Alternative to find_changed() given explicit changes.
+
+        This only calls self.fs.stat() on added or updated files, not
+        on all files.  It believes all other files are unchanged!
+
+        Implies add_watched_paths() for add and update, and
+        remove_watched_paths() for remove.
+        """
+        self.remove_watched_paths(remove)
+        self.add_watched_paths(update)
+        return self._find_changed(update)

mypy/gclogger.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import gc
+import time
+from typing import Mapping
+
+
+class GcLogger:
+    """Context manager to log GC stats and overall time."""
+
+    def __enter__(self) -> GcLogger:
+        self.gc_start_time: float | None = None
+        self.gc_time = 0.0
+        self.gc_calls = 0
+        self.gc_collected = 0
+        self.gc_uncollectable = 0
+        gc.callbacks.append(self.gc_callback)
+        self.start_time = time.time()
+        return self
+
+    def gc_callback(self, phase: str, info: Mapping[str, int]) -> None:
+        if phase == "start":
+            assert self.gc_start_time is None, "Start phase out of sequence"
+            self.gc_start_time = time.time()
+        elif phase == "stop":
+            assert self.gc_start_time is not None, "Stop phase out of sequence"
+            self.gc_calls += 1
+            self.gc_time += time.time() - self.gc_start_time
+            self.gc_start_time = None
+            self.gc_collected += info["collected"]
+            self.gc_uncollectable += info["uncollectable"]
+        else:
+            assert False, f"Unrecognized gc phase ({phase!r})"
+
+    def __exit__(self, *args: object) -> None:
+        while self.gc_callback in gc.callbacks:
+            gc.callbacks.remove(self.gc_callback)
+
+    def get_stats(self) -> Mapping[str, float]:
+        end_time = time.time()
+        result = {}
+        result["gc_time"] = self.gc_time
+        result["gc_calls"] = self.gc_calls
+        result["gc_collected"] = self.gc_collected
+        result["gc_uncollectable"] = self.gc_uncollectable
+        result["build_time"] = end_time - self.start_time
+        return result

mypy/git.py

@@ -0,0 +1,34 @@
+"""Git utilities."""
+
+# Used also from setup.py, so don't pull in anything additional here (like mypy or typing):
+from __future__ import annotations
+
+import os
+import subprocess
+
+
+def is_git_repo(dir: str) -> bool:
+    """Is the given directory version-controlled with git?"""
+    return os.path.exists(os.path.join(dir, ".git"))
+
+
+def have_git() -> bool:
+    """Can we run the git executable?"""
+    try:
+        subprocess.check_output(["git", "--help"])
+        return True
+    except subprocess.CalledProcessError:
+        return False
+    except OSError:
+        return False
+
+
+def git_revision(dir: str) -> bytes:
+    """Get the SHA-1 of the HEAD of a git repository."""
+    return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dir).strip()
+
+
+def is_dirty(dir: str) -> bool:
+    """Check whether a git repository has uncommitted changes."""
+    output = subprocess.check_output(["git", "status", "-uno", "--porcelain"], cwd=dir)
+    return output.strip() != b""

mypy/graph_utils.py

@@ -0,0 +1,112 @@
+"""Helpers for manipulations with graphs."""
+
+from __future__ import annotations
+
+from typing import AbstractSet, Iterable, Iterator, TypeVar
+
+T = TypeVar("T")
+
+
+def strongly_connected_components(
+    vertices: AbstractSet[T], edges: dict[T, list[T]]
+) -> Iterator[set[T]]:
+    """Compute Strongly Connected Components of a directed graph.
+
+    Args:
+      vertices: the labels for the vertices
+      edges: for each vertex, gives the target vertices of its outgoing edges
+
+    Returns:
+      An iterator yielding strongly connected components, each
+      represented as a set of vertices.  Each input vertex will occur
+      exactly once; vertices not part of a SCC are returned as
+      singleton sets.
+
+    From https://code.activestate.com/recipes/578507/.
+    """
+    identified: set[T] = set()
+    stack: list[T] = []
+    index: dict[T, int] = {}
+    boundaries: list[int] = []
+
+    def dfs(v: T) -> Iterator[set[T]]:
+        index[v] = len(stack)
+        stack.append(v)
+        boundaries.append(index[v])
+
+        for w in edges[v]:
+            if w not in index:
+                yield from dfs(w)
+            elif w not in identified:
+                while index[w] < boundaries[-1]:
+                    boundaries.pop()
+
+        if boundaries[-1] == index[v]:
+            boundaries.pop()
+            scc = set(stack[index[v] :])
+            del stack[index[v] :]
+            identified.update(scc)
+            yield scc
+
+    for v in vertices:
+        if v not in index:
+            yield from dfs(v)
+
+
+def prepare_sccs(
+    sccs: list[set[T]], edges: dict[T, list[T]]
+) -> dict[AbstractSet[T], set[AbstractSet[T]]]:
+    """Use original edges to organize SCCs in a graph by dependencies between them."""
+    sccsmap = {v: frozenset(scc) for scc in sccs for v in scc}
+    data: dict[AbstractSet[T], set[AbstractSet[T]]] = {}
+    for scc in sccs:
+        deps: set[AbstractSet[T]] = set()
+        for v in scc:
+            deps.update(sccsmap[x] for x in edges[v])
+        data[frozenset(scc)] = deps
+    return data
+
+
+def topsort(data: dict[T, set[T]]) -> Iterable[set[T]]:
+    """Topological sort.
+
+    Args:
+      data: A map from vertices to all vertices that it has an edge
+            connecting it to.  NOTE: This data structure
+            is modified in place -- for normalization purposes,
+            self-dependencies are removed and entries representing
+            orphans are added.
+
+    Returns:
+      An iterator yielding sets of vertices that have an equivalent
+      ordering.
+
+    Example:
+      Suppose the input has the following structure:
+
+        {A: {B, C}, B: {D}, C: {D}}
+
+      This is normalized to:
+
+        {A: {B, C}, B: {D}, C: {D}, D: {}}
+
+      The algorithm will yield the following values:
+
+        {D}
+        {B, C}
+        {A}
+
+    From https://code.activestate.com/recipes/577413/.
+    """
+    # TODO: Use a faster algorithm?
+    for k, v in data.items():
+        v.discard(k)  # Ignore self dependencies.
+    for item in set.union(*data.values()) - set(data.keys()):
+        data[item] = set()
+    while True:
+        ready = {item for item, dep in data.items() if not dep}
+        if not ready:
+            break
+        yield ready
+        data = {item: (dep - ready) for item, dep in data.items() if item not in ready}
+    assert not data, f"A cyclic dependency exists amongst {data!r}"