execsql2 2.29.0__py3-none-any.whl → 2.30.0__py3-none-any.whl

execsql/lsp/server.py

@@ -6,6 +6,7 @@ import asyncio
 import logging
 import os
 from dataclasses import dataclass, field
+from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 
@@ -14,10 +15,11 @@ from pygls.lsp.server import LanguageServer
 from pygls.uris import to_fs_path
 
 from execsql import __version__
+from execsql.lsp.cache import IndexCache
 from execsql.lsp.code_actions import code_actions
 from execsql.lsp.completion import completions
-from execsql.lsp.diagnostics import diagnostics, too_deep_diagnostic
-from execsql.lsp.document import ScriptIndex, index_script, read_included, same_file_key
+from execsql.lsp.diagnostics import diagnostics, failure_diagnostic, too_deep_diagnostic
+from execsql.lsp.document import ScriptIndex, index_script, same_file_key
 from execsql.lsp.folding import folding_ranges
 from execsql.lsp.formatting import format_document
 from execsql.lsp.hover import hover
@@ -26,6 +28,8 @@ from execsql.lsp.navigation import definition, document_links, document_symbols,
 
 __all__ = ["ExecsqlLanguageServer", "create_server"]
 
+_log = logging.getLogger(__name__)
+
 # Pause after the last keystroke before linting, so a burst of typing lints once.
 LINT_DELAY_SECONDS = 0.2
 
@@ -54,34 +58,42 @@ class Settings:
     select: tuple[str, ...] = ()
     ignore: tuple[str, ...] = ()
     format_options: dict[str, Any] = field(default_factory=dict)  # format_document keywords, from [format]
+    config_files: tuple[str, ...] = ()  # the files read, lowest precedence first
 
 
 class ExecsqlLanguageServer(LanguageServer):
-    """Holds what the handlers share: each workspace folder's settings and pending lint tasks."""
+    """Holds what the handlers share: each workspace folder's settings, pending lint tasks and the index cache."""
 
     def __init__(self) -> None:
         super().__init__("execsql", __version__)
         self.settings: dict[str, Settings] = {}  # by same_file_key of each workspace folder
         self.fallback = Settings()  # for a file in no workspace folder
         self.pending: dict[str, asyncio.Task[Any]] = {}
+        self.cache = IndexCache()
 
     def load_settings(self) -> None:
         """Read ``[lint]`` and ``[format]`` for each workspace folder, as ``execsql lint`` / ``execsql format`` run there would.
 
-        execsql reads config files from the working directory, so each
-        folder's settings are read from inside it.  The server itself works
-        from the workspace root: a file in no folder gets the settings read
-        there, and an unsaved script's relative ``INCLUDE`` resolves there.
+        Each folder stands in for the working directory those commands read
+        config from; the server's own working directory never changes.  A file
+        in no folder gets the settings of :meth:`base_dir`.
         """
-        root = self.workspace.root_path
-        if root and os.path.isdir(root):
-            os.chdir(root)
-        self.fallback = self._read_settings(None)
+        self.fallback = self._read_settings(self.base_dir())
+        _log_settings("files in no workspace folder", self.fallback)
         self.settings = {}
         for folder in self.workspace.folders.values():
             path = to_fs_path(folder.uri)
             if path and os.path.isdir(path):
-                self.settings[same_file_key(path)] = self._read_settings(path)
+                self.settings[same_file_key(path)] = settings = self._read_settings(path)
+                _log_settings(path, settings)
+
+    def base_dir(self) -> str | None:
+        """The workspace root, if it still exists: an unsaved script's relative paths resolve there.
+
+        ``None`` (no root) leaves them to the server's working directory.
+        """
+        root = self.workspace.root_path
+        return root if root and os.path.isdir(root) else None
 
     def _read_settings(self, folder: str | None) -> Settings:
         """The settings ``execsql lint`` run from *folder* (the working directory if ``None``) reads.
@@ -91,11 +103,8 @@ class ExecsqlLanguageServer(LanguageServer):
         from execsql.cli.lint import resolve_selectors
         from execsql.cli.run import _tool_config
 
-        here = os.getcwd()
         try:
-            if folder is not None:
-                os.chdir(folder)
-            conf = _tool_config(None)
+            conf = _tool_config(None, start_dir=folder)
             return Settings(
                 select=resolve_selectors([conf.lint_select]) if conf.lint_select else (),
                 ignore=resolve_selectors([conf.lint_ignore]) if conf.lint_ignore else (),
@@ -105,14 +114,14 @@ class ExecsqlLanguageServer(LanguageServer):
                     "leading_comma": conf.format_leading_comma,
                     "rewrite_sql": conf.format_rewrite_sql,
                 },
+                config_files=tuple(conf.files_read),
             )
         except Exception as exc:  # a bad config must not take the server down
+            _log.warning("execsql config for %s: %s", folder or "the working directory", exc)
             self.window_show_message(
                 types.ShowMessageParams(type=types.MessageType.Warning, message=f"execsql config: {exc}"),
             )
             return Settings()
-        finally:
-            os.chdir(here)
 
     def settings_for(self, uri: str) -> Settings:
         """The settings of the innermost workspace folder holding *uri*'s file, else :attr:`fallback`."""
@@ -124,7 +133,7 @@ class ExecsqlLanguageServer(LanguageServer):
         return self.settings[max(holding, key=len)] if holding else self.fallback
 
     def index(self, uri: str) -> ScriptIndex:
-        """The current document's index (rebuilt per request: a few milliseconds).
+        """The current document's index, rebuilt per request from :attr:`cache`d per-file facts.
 
         A script the parser cannot handle (nesting past Python's recursion
         limit) gets an empty index, so features fall back to what needs none.
@@ -132,9 +141,12 @@ class ExecsqlLanguageServer(LanguageServer):
         document = self.workspace.get_text_document(uri)
         path = to_fs_path(uri)
         try:
-            index = index_script(document.source, path, self.read_included)
+            index = index_script(document.source, path, self.read_included, self.cache.facts, self.base_dir())
         except RecursionError:
             index = ScriptIndex(path=path)
+        except Exception:  # the features that need an index answer with nothing; the cause goes to the log
+            _log.exception("execsql lsp: could not index %s", uri)
+            index = ScriptIndex(path=path)
         index.open_uris = self.open_uris()
         return index
 
@@ -161,7 +173,7 @@ class ExecsqlLanguageServer(LanguageServer):
         uri = self.open_uris().get(same_file_key(str(path)))
         if uri is not None:
             return self.workspace.get_text_document(uri).source
-        return read_included(path)
+        return self.cache.text(path)
 
     def supports_snippets(self) -> bool:
         caps = self.client_capabilities.text_document
@@ -173,17 +185,41 @@ class ExecsqlLanguageServer(LanguageServer):
         document = self.workspace.get_text_document(uri)
         settings = self.settings_for(uri)
         try:
-            found = diagnostics(document.source, to_fs_path(uri), settings.select, settings.ignore)
+            found = diagnostics(document.source, to_fs_path(uri), settings.select, settings.ignore, self.base_dir())
         except RecursionError:
             found = [too_deep_diagnostic()]
+        except Exception as exc:  # shown in the editor, not lost in an unawaited task
+            _log.exception("execsql lsp: could not lint %s", uri)
+            found = [failure_diagnostic(exc)]
         self.text_document_publish_diagnostics(
             types.PublishDiagnosticsParams(uri=uri, version=document.version, diagnostics=found),
         )
 
-    def publish_all(self) -> None:
-        """Lint every open document again: a file one of them includes, or the config, changed."""
+    def relint(self, changed: Iterable[str] | None = None) -> None:
+        """Lint again, after :data:`LINT_DELAY_SECONDS`, each open document a change to the *changed* files can affect.
+
+        Lint reads no included file's text; it checks only that each file a
+        script ``INCLUDE``s exists.  So a document is affected by a change to
+        itself or to a file it includes.  ``None`` means everything changed
+        (the config, the workspace folders).  Going through the debounce, the
+        editor's save and its file watcher's report of the same save lint once.
+        """
+        keys = None if changed is None else {same_file_key(path) for path in changed}
         for uri in list(self.workspace.text_documents):
-            self.publish(uri)
+            if keys is None or self._affected(uri, keys):
+                self.publish_later(uri)
+
+    def _affected(self, uri: str, keys: set[str]) -> bool:
+        path = to_fs_path(uri)
+        if path is None:
+            return False
+        if same_file_key(path) in keys:
+            return True
+        try:
+            facts = self.cache.facts(self.workspace.get_text_document(uri).source, path)
+        except Exception:  # when unsure, lint again
+            return True
+        return any(same_file_key(target) in keys for target in facts.includes)
 
     def watch_files(self) -> None:
         """Ask the client to report changes to :data:`WATCHED_FILES`, if it can."""
@@ -219,6 +255,36 @@ class ExecsqlLanguageServer(LanguageServer):
         self.pending[uri] = asyncio.ensure_future(run())
 
 
+def _log_settings(where: str, settings: Settings) -> None:
+    _log.info(
+        "settings for %s: config files %s; lint select %s, ignore %s; format %s",
+        where,
+        ", ".join(settings.config_files) or "none",
+        ",".join(settings.select) or "all",
+        ",".join(settings.ignore) or "none",
+        settings.format_options or "defaults",
+    )
+
+
+def startup_record() -> str:
+    """What a bug report needs first: which execsql, run by which Python, with which optional parts."""
+    import sys
+    from importlib.metadata import PackageNotFoundError, version
+    from importlib.util import find_spec
+
+    def installed(name: str) -> str:
+        try:
+            return version(name)
+        except PackageNotFoundError:
+            return "not installed"
+
+    formatter = "available" if find_spec("sqlglot") is not None else "not installed (Format Document needs it)"
+    return (
+        f"execsql lsp {__version__}; Python {sys.version.split()[0]} at {sys.executable}; "
+        f"pygls {installed('pygls')}; formatter {formatter}"
+    )
+
+
 def create_server() -> ExecsqlLanguageServer:
     """A server with every feature registered."""
     server = ExecsqlLanguageServer()
@@ -228,9 +294,19 @@ def create_server() -> ExecsqlLanguageServer:
 
     @server.feature(types.INITIALIZED)
     def initialized(ls: ExecsqlLanguageServer, params: types.InitializedParams) -> None:
+        _log.info(startup_record())
+        folders = [to_fs_path(f.uri) or f.uri for f in ls.workspace.folders.values()]
+        _log.info("workspace folders: %s", ", ".join(folders) or "none")
         ls.load_settings()
         ls.watch_files()
 
+    @server.feature(types.SHUTDOWN)
+    def shutdown(ls: ExecsqlLanguageServer, params: None) -> None:
+        # No diagnostics may follow the answer to shutdown.
+        for task in ls.pending.values():
+            task.cancel()
+        ls.pending.clear()
+
     @server.feature(types.TEXT_DOCUMENT_DID_OPEN)
     def did_open(ls: ExecsqlLanguageServer, params: types.DidOpenTextDocumentParams) -> None:
         ls.publish(params.text_document.uri)
@@ -242,19 +318,25 @@ def create_server() -> ExecsqlLanguageServer:
     @server.feature(types.TEXT_DOCUMENT_DID_SAVE)
     def did_save(ls: ExecsqlLanguageServer, params: types.DidSaveTextDocumentParams) -> None:
         # Another open script may INCLUDE this one; a client that cannot watch files reports nothing else.
-        ls.publish_all()
+        path = to_fs_path(params.text_document.uri)
+        ls.relint([path] if path else [])
 
     @server.feature(types.WORKSPACE_DID_CHANGE_WATCHED_FILES)
     def did_change_watched_files(ls: ExecsqlLanguageServer, params: types.DidChangeWatchedFilesParams) -> None:
-        if any(Path(to_fs_path(change.uri) or "").name.lower() == "execsql.conf" for change in params.changes):
+        paths = [path for change in params.changes if (path := to_fs_path(change.uri))]
+        for path in paths:
+            ls.cache.forget(path)
+        if any(Path(path).name.lower() == "execsql.conf" for path in paths):
             ls.load_settings()
-        ls.publish_all()
+            ls.relint()
+        else:
+            ls.relint(paths)
 
     @server.feature(types.WORKSPACE_DID_CHANGE_WORKSPACE_FOLDERS)
     def did_change_folders(ls: ExecsqlLanguageServer, params: types.DidChangeWorkspaceFoldersParams) -> None:
         # pygls has already updated ls.workspace.folders.
         ls.load_settings()
-        ls.publish_all()
+        ls.relint()
 
     @server.feature(types.TEXT_DOCUMENT_DID_CLOSE)
     def did_close(ls: ExecsqlLanguageServer, params: types.DidCloseTextDocumentParams) -> None:

execsql/manifest.py

@@ -10,7 +10,11 @@ Files are recorded as the executor runs each metacommand, after variable
 substitution, so paths are the ones actually used. What a metacommand touches
 is decided by :mod:`execsql.metacommands.effects`. Variable values and passwords are never written:
 the run log hides ``-a`` and ``--var`` values because they can be secrets, and
-the manifest lists only their names.
+the manifest lists only their names. A password or decrypted value that reaches
+a path, a connection target or an error message is written as ``***``, as are
+common secret shapes (a URL's password, ``password=...``, API tokens). ``-a`` and
+``--var`` values, which the log also masks, are kept: they are usually part of
+the paths the manifest exists to report.
 
 One recorder is active per run; :func:`current` returns it, or ``None`` when
 no manifest was asked for, so the executor's hooks cost one check. Writing the
@@ -81,6 +85,22 @@ def _now() -> datetime.datetime:
     return datetime.datetime.now(tz=datetime.timezone.utc)
 
 
+def _masked(value: Any) -> Any:
+    """*value* with the run's passwords and common secret shapes replaced by ``***`` in every string it holds.
+
+    ``-a`` / ``--var`` values are left in place; times, counts and line numbers are not strings.
+    """
+    from execsql.utils.fileio import redact
+
+    if isinstance(value, str):
+        return redact(value, arguments=False)
+    if isinstance(value, list):
+        return [_masked(item) for item in value]
+    if isinstance(value, dict):
+        return {key: _masked(item) for key, item in value.items()}
+    return value
+
+
 class RunManifest:
     """Collects what one run does and writes it as JSON when the run ends."""
 
@@ -166,20 +186,20 @@ class RunManifest:
         finished = _now()
         data = {
             "execsql_version": __version__,
-            "script": self.script,
+            "script": _masked(self.script),
             "started": self.started.isoformat(timespec="seconds"),
             "finished": finished.isoformat(timespec="seconds"),
             "duration_s": round((finished - self.started).total_seconds(), 3),
             "exit_status": exit_status,
-            "config_files": self.config_files,
-            "database": self.database,
+            "config_files": _masked(self.config_files),
+            "database": _masked(self.database),
             "variables": self.variables,
             "statements": {"sql": self.sql_statements, "metacommands": self.metacommands},
-            "files_read": self.reads,
-            "files_written": self.writes,
-            "files_deleted": self.deletes,
-            "connections": self.connections,
-            "errors": self.errors,
+            "files_read": _masked(self.reads),
+            "files_written": _masked(self.writes),
+            "files_deleted": _masked(self.deletes),
+            "connections": _masked(self.connections),
+            "errors": _masked(self.errors),
         }
         self.written = True
         import execsql.state as _state

execsql/metacommands/__init__.py

@@ -242,7 +242,6 @@ from execsql.metacommands.system import (
 # Regex helper functions (from utils/regex.py)
 from execsql.utils.regex import (
     ins_rxs,
-    ins_quoted_rx,
     ins_schema_rxs,
     ins_table_rxs,
     ins_table_list_rxs,
@@ -443,7 +442,6 @@ __all__ = [
     "x_pg_upsert_qa",
     # regex helpers
     "ins_rxs",
-    "ins_quoted_rx",
     "ins_schema_rxs",
     "ins_table_rxs",
     "ins_table_list_rxs",

execsql/metacommands/io_fileops.py

@@ -307,6 +307,11 @@ def x_cd(**kwargs: Any) -> None:
             command_text=kwargs["metacommandline"],
             other_msg="Directory does not exist",
         )
+    if _state.cd_from is None:
+        try:
+            _state.cd_from = os.getcwd()
+        except OSError:  # the directory is gone: nowhere to return to
+            pass
     os.chdir(new_dir)
     script, lno = current_script_line()
     _state.exec_log.log_status_info(f"Current directory changed to {new_dir} at line {lno} of {script}")

execsql/script/executor.py

@@ -957,16 +957,15 @@ class ExecutingStatement:
     rather than copying fields out of it, so there is nothing to keep in sync
     and no second source of truth about what is executing.  Error reporting
     (:mod:`execsql.utils.errors`), the debug REPL, and ``api.run()`` read
-    these five attributes; everything is derived from ``node`` on access, so
+    its attributes; everything is derived from ``node`` on access, so
     constructing one costs a single small object per statement rather than
     the two classes the previous shim built each time.
     """
 
-    __slots__ = ("node", "_source_dir")
+    __slots__ = ("node",)
 
     def __init__(self, node: Node) -> None:
         self.node = node
-        self._source_dir: str | None = None
 
     @property
     def source(self) -> str:
@@ -976,18 +975,6 @@ class ExecutingStatement:
     def line_no(self) -> int:
         return self.node.span.start_line
 
-    @property
-    def source_name(self) -> str:
-        return Path(self.node.span.file).name
-
-    @property
-    def source_dir(self) -> str:
-        # resolve() touches the filesystem, so it is done once per statement
-        # object rather than on every read.
-        if self._source_dir is None:
-            self._source_dir = str(Path(self.node.span.file).resolve().parent) + os.sep
-        return self._source_dir
-
     @property
     def command_type(self) -> str:
         return "sql" if isinstance(self.node, SqlStatement) else "cmd"
@@ -1004,12 +991,6 @@ class ExecutingStatement:
     def __repr__(self) -> str:
         return f"ExecutingStatement({self.source}:{self.line_no}, {self.command_type})"
 
-    def current_script_line(self) -> tuple[str, int]:
-        return (self.source, self.line_no)
-
-    def commandline(self) -> str:
-        return cast(str, self.command.commandline())
-
 
 # ---------------------------------------------------------------------------
 # Node type/text helpers for profiling

execsql/state.py

@@ -89,8 +89,6 @@ class ExecFrame:
 
 
 if TYPE_CHECKING:
-    import multiprocessing as _mp
-
     from execsql.config import ConfigData, StatObj, WriteHooks
     from execsql.db.base import DatabasePool
     from execsql.exporters.base import ExportMetadata, WriteSpec
@@ -230,6 +228,7 @@ _CONTEXT_ATTRS: frozenset[str] = frozenset(
         "manifest",
         "halt_error",
         "halt_output_error",
+        "cd_from",
         # Lazy singletons
         "counters",
         "timer",
@@ -288,6 +287,7 @@ class RuntimeContext:
         "manifest",
         "halt_error",
         "halt_output_error",
+        "cd_from",
         # Lazy singletons
         "counters",
         "timer",
@@ -344,6 +344,9 @@ class RuntimeContext:
         self.halt_error: ErrInfo | None = None
         # File output lost when the run halted (set by ``exit_now``), or None.
         self.halt_output_error: ErrInfo | None = None
+        # The working directory before this run's first CD, or None: the
+        # library's run() returns its caller there.
+        self.cd_from: str | None = None
 
         # Lazy singletons
         self.counters: CounterVars | None = None
@@ -356,7 +359,7 @@ class RuntimeContext:
 
         # GUI
         self.gui_console: Any = None
-        self.gui_manager_queue: _mp.Queue | None = None
+        self.gui_manager_queue: Any = None  # a queue.Queue or a _TkinterSyncQueue; only put() is used
         self.gui_manager_thread: threading.Thread | None = None
 
         # Profiling — None means profiling is disabled; a list means it is enabled.

execsql/state.pyi

@@ -59,6 +59,7 @@ filewriter: FileWriter | None
 manifest: RunManifest | None
 halt_error: ErrInfo | None
 halt_output_error: ErrInfo | None
+cd_from: str | None
 counters: CounterVars
 timer: Timer
 dbs: DatabasePool
@@ -97,6 +98,7 @@ class RuntimeContext:
     manifest: RunManifest | None
     halt_error: ErrInfo | None
     halt_output_error: ErrInfo | None
+    cd_from: str | None
     counters: CounterVars | None
     timer: Timer | None
     dbs: DatabasePool | None

execsql/utils/errors.py

@@ -6,8 +6,6 @@ Error handling and reporting utilities for execsql.
 Provides the functions that form the error-reporting pipeline used
 throughout the codebase:
 
-- :func:`exception_info` — extracts type, value, and source location
-  from the current exception.
 - :func:`exception_desc` — ``Type: message`` for the exception being
   handled, as shown on screen.
 - :func:`error_origin` / :func:`logged_error` — where in execsql an error
@@ -28,7 +26,6 @@ import execsql.state as _state
 from execsql.exceptions import ErrInfo
 
 __all__ = [
-    "exception_info",
     "error_origin",
     "exception_desc",
     "logged_error",
@@ -56,43 +53,29 @@ def install_hint(extra: str) -> str:
     )
 
 
-def exception_info() -> tuple:
-    # Returns the exception type, value, source file name, source line number, and source line text.
-    strace = traceback.extract_tb(sys.exc_info()[2])[-1:]
-    traces = traceback.extract_tb(sys.exc_info()[2])
-    xline = 0
-    for trace in traces:
-        if "execsql" in trace[0]:
-            xline = trace[1]
-    exc_message = ""
-    exc_param = sys.exc_info()[1]
-    if isinstance(exc_param, str):
-        exc_message = exc_param
-    elif exc_param is not None:
-        message = getattr(exc_param, "message", None)
-        value = getattr(exc_param, "value", None)
-        if isinstance(message, str) and len(message) > 0:
-            exc_message = message
-        elif isinstance(value, str) and len(value) > 0:
-            exc_message = value
-        else:
-            exc_message = str(exc_param)
-    try:
-        exc_message = str(exc_message)
-    except Exception:
-        exc_message = repr(exc_message)
-    xinfo = sys.exc_info()[0]
-    xname = getattr(xinfo, "__name__", "")
-    return xname, exc_message, strace[0][0], xline, strace[0][3]
-
-
 def exception_desc() -> str:
     """``Type: message`` for the exception being handled, as shown to the person running the script.
 
-    Where in execsql it was raised is for the log: see :func:`error_origin`.
+    The message is the exception's ``message`` or ``value`` attribute when it
+    has a non-empty string one (as some drivers' exceptions do), else its
+    ``str()``.  Where in execsql it was raised is for the log: see
+    :func:`error_origin`.
     """
-    exc_type, exc_strval, _filename, _lineno, _linetext = exception_info()
-    return f"{exc_type}: {exc_strval}"
+    exc_type, exc = sys.exc_info()[:2]
+    if isinstance(exc, str):
+        message = exc
+    elif exc is None:
+        message = ""
+    else:
+        attr = next(
+            (v for v in (getattr(exc, "message", None), getattr(exc, "value", None)) if isinstance(v, str) and v),
+            None,
+        )
+        try:
+            message = attr if attr is not None else str(exc)
+        except Exception:
+            message = repr(exc)
+    return f"{getattr(exc_type, '__name__', '')}: {message}"
 
 
 def error_origin(exc: BaseException) -> str | None: