streamlit-nightly 1.43.2.dev20250307__py3-none-any.whl

streamlit/watcher/local_sources_watcher.py

@@ -0,0 +1,233 @@
+# Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022-2025)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Final, NamedTuple
+
+from streamlit import config, file_util
+from streamlit.logger import get_logger
+from streamlit.watcher.folder_black_list import FolderBlackList
+from streamlit.watcher.path_watcher import (
+    NoOpPathWatcher,
+    get_default_path_watcher_class,
+)
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+    from streamlit.runtime.pages_manager import PagesManager
+
+_LOGGER: Final = get_logger(__name__)
+
+
+class WatchedModule(NamedTuple):
+    watcher: Any
+    module_name: Any
+
+
+# This needs to be initialized lazily to avoid calling config.get_option() and
+# thus initializing config options when this file is first imported.
+PathWatcher = None
+
+
+class LocalSourcesWatcher:
+    def __init__(self, pages_manager: PagesManager):
+        self._pages_manager = pages_manager
+        self._main_script_path = os.path.abspath(self._pages_manager.main_script_path)
+        self._script_folder = os.path.dirname(self._main_script_path)
+        self._on_file_changed: list[Callable[[str], None]] = []
+        self._is_closed = False
+        self._cached_sys_modules: set[str] = set()
+
+        # Blacklist for folders that should not be watched
+        self._folder_black_list = FolderBlackList(
+            config.get_option("server.folderWatchBlacklist")
+        )
+
+        self._watched_modules: dict[str, WatchedModule] = {}
+        self._watched_pages: set[str] = set()
+
+        self.update_watched_pages()
+
+    def update_watched_pages(self) -> None:
+        old_page_paths = self._watched_pages.copy()
+        new_pages_paths: set[str] = set()
+
+        for page_info in self._pages_manager.get_pages().values():
+            if not page_info["script_path"]:
+                continue
+
+            new_pages_paths.add(page_info["script_path"])
+            if page_info["script_path"] not in self._watched_pages:
+                self._register_watcher(
+                    page_info["script_path"],
+                    module_name=None,
+                )
+
+        for old_page_path in old_page_paths:
+            # Only remove pages that are no longer valid files
+            if old_page_path not in new_pages_paths and not os.path.isfile(
+                old_page_path
+            ):
+                self._deregister_watcher(old_page_path)
+                self._watched_pages.remove(old_page_path)
+
+        self._watched_pages = self._watched_pages.union(new_pages_paths)
+
+    def register_file_change_callback(self, cb: Callable[[str], None]) -> None:
+        self._on_file_changed.append(cb)
+
+    def on_file_changed(self, filepath):
+        if filepath not in self._watched_modules:
+            _LOGGER.error("Received event for non-watched file: %s", filepath)
+            return
+
+        # Workaround:
+        # Delete all watched modules so we can guarantee changes to the
+        # updated module are reflected on reload.
+        #
+        # In principle, for reloading a given module, we only need to unload
+        # the module itself and all of the modules which import it (directly
+        # or indirectly) such that when we exec the application code, the
+        # changes are reloaded and reflected in the running application.
+        #
+        # However, determining all import paths for a given loaded module is
+        # non-trivial, and so as a workaround we simply unload all watched
+        # modules.
+        for wm in self._watched_modules.values():
+            if wm.module_name is not None and wm.module_name in sys.modules:
+                del sys.modules[wm.module_name]
+
+        for cb in self._on_file_changed:
+            cb(filepath)
+
+    def close(self):
+        for wm in self._watched_modules.values():
+            wm.watcher.close()
+        self._watched_modules = {}
+        self._watched_pages = set()
+        self._is_closed = True
+
+    def _register_watcher(self, filepath, module_name):
+        global PathWatcher
+        if PathWatcher is None:
+            PathWatcher = get_default_path_watcher_class()
+
+        if PathWatcher is NoOpPathWatcher:
+            return
+
+        try:
+            wm = WatchedModule(
+                watcher=PathWatcher(filepath, self.on_file_changed),
+                module_name=module_name,
+            )
+        except PermissionError:
+            # If you don't have permission to read this file, don't even add it
+            # to watchers.
+            return
+
+        self._watched_modules[filepath] = wm
+
+    def _deregister_watcher(self, filepath):
+        if filepath not in self._watched_modules:
+            return
+
+        if filepath == self._main_script_path:
+            return
+
+        wm = self._watched_modules[filepath]
+        wm.watcher.close()
+        del self._watched_modules[filepath]
+
+    def _file_is_new(self, filepath):
+        return filepath not in self._watched_modules
+
+    def _file_should_be_watched(self, filepath):
+        # Using short circuiting for performance.
+        return self._file_is_new(filepath) and (
+            file_util.file_is_in_folder_glob(filepath, self._script_folder)
+            or file_util.file_in_pythonpath(filepath)
+        )
+
+    def update_watched_modules(self):
+        if self._is_closed:
+            return
+
+        if set(sys.modules) != self._cached_sys_modules:
+            modules_paths = {
+                name: self._exclude_blacklisted_paths(get_module_paths(module))
+                for name, module in dict(sys.modules).items()
+            }
+            self._cached_sys_modules = set(sys.modules)
+            self._register_necessary_watchers(modules_paths)
+
+    def _register_necessary_watchers(self, module_paths: dict[str, set[str]]) -> None:
+        for name, paths in module_paths.items():
+            for path in paths:
+                if self._file_should_be_watched(path):
+                    self._register_watcher(str(Path(path).resolve()), name)
+
+    def _exclude_blacklisted_paths(self, paths: set[str]) -> set[str]:
+        return {p for p in paths if not self._folder_black_list.is_blacklisted(p)}
+
+
+def get_module_paths(module: ModuleType) -> set[str]:
+    paths_extractors = [
+        # https://docs.python.org/3/reference/datamodel.html
+        # __file__ is the pathname of the file from which the module was loaded
+        # if it was loaded from a file.
+        # The __file__ attribute may be missing for certain types of modules
+        lambda m: [m.__file__],
+        # https://docs.python.org/3/reference/import.html#__spec__
+        # The __spec__ attribute is set to the module spec that was used
+        # when importing the module. one exception is __main__,
+        # where __spec__ is set to None in some cases.
+        # https://www.python.org/dev/peps/pep-0451/#id16
+        # "origin" in an import context means the system
+        # (or resource within a system) from which a module originates
+        # ... It is up to the loader to decide on how to interpret
+        # and use a module's origin, if at all.
+        lambda m: [m.__spec__.origin],
+        # https://www.python.org/dev/peps/pep-0420/
+        # Handling of "namespace packages" in which the __path__ attribute
+        # is a _NamespacePath object with a _path attribute containing
+        # the various paths of the package.
+        lambda m: list(m.__path__._path),
+    ]
+
+    all_paths = set()
+    for extract_paths in paths_extractors:
+        potential_paths = []
+        try:
+            potential_paths = extract_paths(module)
+        except AttributeError:
+            # Some modules might not have __file__ or __spec__ attributes.
+            pass
+        except Exception:
+            _LOGGER.warning(
+                f"Examining the path of {module.__name__} raised:", exc_info=True
+            )
+
+        all_paths.update(
+            [os.path.abspath(str(p)) for p in potential_paths if _is_valid_path(p)]
+        )
+    return all_paths
+
+
+def _is_valid_path(path: str | None) -> bool:
+    return isinstance(path, str) and (os.path.isfile(path) or os.path.isdir(path))

streamlit/watcher/path_watcher.py

@@ -0,0 +1,185 @@
+# Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022-2025)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+from typing import Callable, Union
+
+import streamlit.watcher
+from streamlit import cli_util, config, env_util
+from streamlit.watcher.polling_path_watcher import PollingPathWatcher
+
+
+# local_sources_watcher.py caches the return value of
+# get_default_path_watcher_class(), so it needs to differentiate between the
+# cases where it:
+#   1. has yet to call get_default_path_watcher_class()
+#   2. has called get_default_path_watcher_class(), which returned that no
+#      path watcher should be installed.
+# This forces us to define this stub class since the cached value equaling
+# None corresponds to case 1 above.
+class NoOpPathWatcher:
+    def __init__(
+        self,
+        _path_str: str,
+        _on_changed: Callable[[str], None],
+        *,  # keyword-only arguments:
+        glob_pattern: str | None = None,
+        allow_nonexistent: bool = False,
+    ):
+        pass
+
+
+# EventBasedPathWatcher will be a stub and have no functional
+# implementation if its import failed (due to missing watchdog module),
+# so we can't reference it directly in this type.
+PathWatcherType = Union[
+    type["streamlit.watcher.event_based_path_watcher.EventBasedPathWatcher"],
+    type[PollingPathWatcher],
+    type[NoOpPathWatcher],
+]
+
+
+def _is_watchdog_available() -> bool:
+    """Check if the watchdog module is installed."""
+    try:
+        import watchdog  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def report_watchdog_availability():
+    if (
+        config.get_option("server.fileWatcherType") not in ["poll", "none"]
+        and not _is_watchdog_available()
+    ):
+        msg = "\n  $ xcode-select --install" if env_util.IS_DARWIN else ""
+
+        cli_util.print_to_cli(
+            "  %s" % "For better performance, install the Watchdog module:",
+            fg="blue",
+            bold=True,
+        )
+        cli_util.print_to_cli(
+            """%s
+  $ pip install watchdog
+            """
+            % msg
+        )
+
+
+def _watch_path(
+    path: str,
+    on_path_changed: Callable[[str], None],
+    watcher_type: str | None = None,
+    *,  # keyword-only arguments:
+    glob_pattern: str | None = None,
+    allow_nonexistent: bool = False,
+) -> bool:
+    """Create a PathWatcher for the given path if we have a viable
+    PathWatcher class.
+
+    Parameters
+    ----------
+    path
+        Path to watch.
+    on_path_changed
+        Function that's called when the path changes.
+    watcher_type
+        Optional watcher_type string. If None, it will default to the
+        'server.fileWatcherType` config option.
+    glob_pattern
+        Optional glob pattern to use when watching a directory. If set, only
+        files matching the pattern will be counted as being created/deleted
+        within the watched directory.
+    allow_nonexistent
+        If True, allow the file or directory at the given path to be
+        nonexistent.
+
+    Returns
+    -------
+    bool
+        True if the path is being watched, or False if we have no
+        PathWatcher class.
+    """
+    if watcher_type is None:
+        watcher_type = config.get_option("server.fileWatcherType")
+
+    watcher_class = get_path_watcher_class(watcher_type)
+    if watcher_class is NoOpPathWatcher:
+        return False
+
+    watcher_class(
+        path,
+        on_path_changed,
+        glob_pattern=glob_pattern,
+        allow_nonexistent=allow_nonexistent,
+    )
+    return True
+
+
+def watch_file(
+    path: str,
+    on_file_changed: Callable[[str], None],
+    watcher_type: str | None = None,
+) -> bool:
+    return _watch_path(path, on_file_changed, watcher_type)
+
+
+def watch_dir(
+    path: str,
+    on_dir_changed: Callable[[str], None],
+    watcher_type: str | None = None,
+    *,  # keyword-only arguments:
+    glob_pattern: str | None = None,
+    allow_nonexistent: bool = False,
+) -> bool:
+    # Add a trailing slash to the path to ensure
+    # that its interpreted as a directory.
+    path = os.path.join(path, "")
+
+    return _watch_path(
+        path,
+        on_dir_changed,
+        watcher_type,
+        glob_pattern=glob_pattern,
+        allow_nonexistent=allow_nonexistent,
+    )
+
+
+def get_default_path_watcher_class() -> PathWatcherType:
+    """Return the class to use for path changes notifications, based on the
+    server.fileWatcherType config option.
+    """
+    return get_path_watcher_class(config.get_option("server.fileWatcherType"))
+
+
+def get_path_watcher_class(watcher_type: str) -> PathWatcherType:
+    """Return the PathWatcher class that corresponds to the given watcher_type
+    string. Acceptable values are 'auto', 'watchdog', 'poll' and 'none'.
+    """
+    if watcher_type in {"watchdog", "auto"} and _is_watchdog_available():
+        # Lazy-import this module to prevent unnecessary imports of the watchdog package.
+        from streamlit.watcher.event_based_path_watcher import EventBasedPathWatcher
+
+        return EventBasedPathWatcher
+    elif watcher_type == "auto":
+        return PollingPathWatcher
+    elif watcher_type == "poll":
+        return PollingPathWatcher
+    else:
+        return NoOpPathWatcher

streamlit/watcher/polling_path_watcher.py

@@ -0,0 +1,124 @@
+# Copyright (c) Streamlit Inc. (2018-2022) Snowflake Inc. (2022-2025)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A class that watches a given path via polling."""
+
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable, Final
+
+from streamlit.logger import get_logger
+from streamlit.util import repr_
+from streamlit.watcher import util
+
+_LOGGER: Final = get_logger(__name__)
+
+_MAX_WORKERS: Final = 4
+_POLLING_PERIOD_SECS: Final = 0.2
+
+
+class PollingPathWatcher:
+    """Watches a path on disk via a polling loop."""
+
+    _executor = ThreadPoolExecutor(max_workers=_MAX_WORKERS)
+
+    @staticmethod
+    def close_all() -> None:
+        """Close top-level watcher object.
+
+        This is a no-op, and exists for interface parity with
+        EventBasedPathWatcher.
+        """
+        _LOGGER.debug("Watcher closed")
+
+    def __init__(
+        self,
+        path: str,
+        on_changed: Callable[[str], None],
+        *,  # keyword-only arguments:
+        glob_pattern: str | None = None,
+        allow_nonexistent: bool = False,
+    ) -> None:
+        """Constructor.
+
+        You do not need to retain a reference to a PollingPathWatcher to
+        prevent it from being garbage collected. (The global _executor object
+        retains references to all active instances.)
+        """
+        # TODO(vdonato): Modernize this by switching to pathlib.
+        self._path = path
+        self._on_changed = on_changed
+
+        self._glob_pattern = glob_pattern
+        self._allow_nonexistent = allow_nonexistent
+
+        self._active = True
+
+        self._modification_time = util.path_modification_time(
+            self._path, self._allow_nonexistent
+        )
+        self._md5 = util.calc_md5_with_blocking_retries(
+            self._path,
+            glob_pattern=self._glob_pattern,
+            allow_nonexistent=self._allow_nonexistent,
+        )
+        self._schedule()
+
+    def __repr__(self) -> str:
+        return repr_(self)
+
+    def _schedule(self) -> None:
+        def task():
+            time.sleep(_POLLING_PERIOD_SECS)
+            self._check_if_path_changed()
+
+        PollingPathWatcher._executor.submit(task)
+
+    def _check_if_path_changed(self) -> None:
+        if not self._active:
+            # Don't call self._schedule()
+            return
+
+        modification_time = util.path_modification_time(
+            self._path, self._allow_nonexistent
+        )
+        # We add modification_time != 0.0 check since on some file systems (s3fs/fuse)
+        # modification_time is always 0.0 because of file system limitations.
+        if modification_time != 0.0 and modification_time <= self._modification_time:
+            self._schedule()
+            return
+
+        self._modification_time = modification_time
+
+        md5 = util.calc_md5_with_blocking_retries(
+            self._path,
+            glob_pattern=self._glob_pattern,
+            allow_nonexistent=self._allow_nonexistent,
+        )
+        if md5 == self._md5:
+            self._schedule()
+            return
+
+        self._md5 = md5
+
+        _LOGGER.debug("Change detected: %s", self._path)
+        self._on_changed(self._path)
+
+        self._schedule()
+
+    def close(self) -> None:
+        """Stop watching the file system."""
+        self._active = False