robotcode-robot 0.94.0__py3-none-any.whl → 0.95.1__py3-none-any.whl

robotcode/robot/__version__.py

@@ -1 +1 @@
-__version__ = "0.94.0"
+__version__ = "0.95.1"

robotcode/robot/diagnostics/data_cache.py

@@ -0,0 +1,83 @@
+import pickle
+from abc import ABC, abstractmethod
+from enum import Enum
+from pathlib import Path
+from typing import Any, Tuple, Type, TypeVar, Union, cast
+
+from robotcode.core.utils.dataclasses import as_json, from_json
+
+_T = TypeVar("_T")
+
+
+class CacheSection(Enum):
+    LIBRARY = "libdoc"
+    VARIABLES = "variables"
+
+
+class DataCache(ABC):
+    @abstractmethod
+    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool: ...
+
+    @abstractmethod
+    def read_cache_data(
+        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
+    ) -> _T: ...
+
+    @abstractmethod
+    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None: ...
+
+
+class JsonDataCache(DataCache):
+    def __init__(self, cache_dir: Path) -> None:
+        self.cache_dir = cache_dir
+
+    def build_cache_data_filename(self, section: CacheSection, entry_name: str) -> Path:
+        return self.cache_dir / section.value / (entry_name + ".json")
+
+    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool:
+        cache_file = self.build_cache_data_filename(section, entry_name)
+        return cache_file.exists()
+
+    def read_cache_data(
+        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
+    ) -> _T:
+        cache_file = self.build_cache_data_filename(section, entry_name)
+        return from_json(cache_file.read_text("utf-8"), types)
+
+    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None:
+        cached_file = self.build_cache_data_filename(section, entry_name)
+
+        cached_file.parent.mkdir(parents=True, exist_ok=True)
+        cached_file.write_text(as_json(data), "utf-8")
+
+
+class PickleDataCache(DataCache):
+    def __init__(self, cache_dir: Path) -> None:
+        self.cache_dir = cache_dir
+
+    def build_cache_data_filename(self, section: CacheSection, entry_name: str) -> Path:
+        return self.cache_dir / section.value / (entry_name + ".pkl")
+
+    def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool:
+        cache_file = self.build_cache_data_filename(section, entry_name)
+        return cache_file.exists()
+
+    def read_cache_data(
+        self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]]
+    ) -> _T:
+        cache_file = self.build_cache_data_filename(section, entry_name)
+
+        with cache_file.open("rb") as f:
+            result = pickle.load(f)
+
+            if isinstance(result, types):
+                return cast(_T, result)
+
+            raise TypeError(f"Expected {types} but got {type(result)}")
+
+    def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None:
+        cached_file = self.build_cache_data_filename(section, entry_name)
+
+        cached_file.parent.mkdir(parents=True, exist_ok=True)
+        with cached_file.open("wb") as f:
+            pickle.dump(data, f)

robotcode/robot/diagnostics/entities.py

@@ -12,11 +12,11 @@ from typing import (
 )
 
 from robot.parsing.lexer.tokens import Token
-from robot.variables.search import search_variable
 from robotcode.core.lsp.types import Position, Range
 from robotcode.robot.utils.match import normalize
 
 from ..utils.ast import range_from_token
+from ..utils.variables import search_variable
 
 if TYPE_CHECKING:
     from robotcode.robot.diagnostics.library_doc import KeywordDoc, LibraryDoc
@@ -145,10 +145,10 @@ class VariableMatcher:
         self.normalized_name = str(normalize(self.base))
 
     def __eq__(self, o: object) -> bool:
-        if isinstance(o, VariableMatcher):
+        if type(o) is VariableMatcher:
             return o.normalized_name == self.normalized_name
 
-        if isinstance(o, str):
+        if type(o) is str:
             match = search_variable(o, "$@&%", ignore_errors=True)
             base = match.base
             if base is None:

robotcode/robot/diagnostics/errors.py

@@ -6,7 +6,7 @@ DIAGNOSTICS_SOURCE_NAME = "robotcode"
 @final
 class Error:
     VARIABLE_NOT_FOUND = "VariableNotFound"
-    ENVIROMMENT_VARIABLE_NOT_FOUND = "EnvirommentVariableNotFound"
+    ENVIRONMENT_VARIABLE_NOT_FOUND = "EnvironmentVariableNotFound"
     KEYWORD_NOT_FOUND = "KeywordNotFound"
     LIBRARY_CONTAINS_NO_KEYWORDS = "LibraryContainsNoKeywords"
     POSSIBLE_CIRCULAR_IMPORT = "PossibleCircularImport"

robotcode/robot/diagnostics/imports_manager.py

@@ -25,6 +25,7 @@ from typing import (
     final,
 )
 
+from robot.libraries import STDLIBS
 from robot.utils.text import split_args_from_name_or_path
 from robotcode.core.concurrent import RLock, run_as_task
 from robotcode.core.documents_manager import DocumentsManager
@@ -35,14 +36,16 @@ from robotcode.core.lsp.types import DocumentUri, FileChangeType, FileEvent
 from robotcode.core.text_document import TextDocument
 from robotcode.core.uri import Uri
 from robotcode.core.utils.caching import SimpleLRUCache
-from robotcode.core.utils.dataclasses import as_json, from_json
-from robotcode.core.utils.glob_path import Pattern, iter_files
+from robotcode.core.utils.glob_path import Pattern
 from robotcode.core.utils.logging import LoggingDescriptor
 from robotcode.core.utils.path import normalized_path, path_is_relative_to
 
 from ..__version__ import __version__
 from ..utils import get_robot_version, get_robot_version_str
 from ..utils.robot_path import find_file_ex
+from ..utils.variables import contains_variable
+from .data_cache import CacheSection
+from .data_cache import PickleDataCache as DefaultDataCache
 from .entities import (
     CommandLineVariableDefinition,
     VariableDefinition,
@@ -521,18 +524,10 @@ class ImportsManager:
         self._logger.trace(lambda: f"use {cache_base_path} as base for caching")
 
         self.cache_path = cache_base_path / ".robotcode_cache"
-
-        self.lib_doc_cache_path = (
-            self.cache_path
-            / f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
-            / get_robot_version_str()
-            / "libdoc"
-        )
-        self.variables_doc_cache_path = (
+        self.data_cache = DefaultDataCache(
             self.cache_path
             / f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
             / get_robot_version_str()
-            / "variables"
         )
 
         self.cmd_variables = variables
@@ -564,9 +559,9 @@ class ImportsManager:
         if environment:
             self._environment.update(environment)
 
-        self._library_files_cache = SimpleLRUCache(1024)
-        self._resource_files_cache = SimpleLRUCache(1024)
-        self._variables_files_cache = SimpleLRUCache(1024)
+        self._library_files_cache = SimpleLRUCache(2048)
+        self._resource_files_cache = SimpleLRUCache(2048)
+        self._variables_files_cache = SimpleLRUCache(2048)
 
         self._executor_lock = RLock(default_timeout=120, name="ImportsManager._executor_lock")
         self._executor: Optional[ProcessPoolExecutor] = None
@@ -582,6 +577,8 @@ class ImportsManager:
             weakref.WeakKeyDictionary()
         )
 
+        self._process_pool_executor: Optional[ProcessPoolExecutor] = None
+
     def __del__(self) -> None:
         try:
             if self._executor is not None:
@@ -899,12 +896,14 @@ class ImportsManager:
                     )
 
             if result is not None:
+                # TODO: use IgnoreSpec instead of this
                 ignore_arguments = any(
                     (p.matches(result.name) if result.name is not None else False)
                     or (p.matches(result.origin) if result.origin is not None else False)
                     for p in self.ignore_arguments_for_library_patters
                 )
 
+                # TODO: use IgnoreSpec instead of this
                 if any(
                     (p.matches(result.name) if result.name is not None else False)
                     or (p.matches(result.origin) if result.origin is not None else False)
@@ -918,16 +917,16 @@ class ImportsManager:
                     return None, import_name, ignore_arguments
 
                 if result.origin is not None:
-                    result.mtimes = {result.origin: Path(result.origin).stat().st_mtime_ns}
+                    result.mtimes = {result.origin: os.stat(result.origin, follow_symlinks=False).st_mtime_ns}
 
                 if result.submodule_search_locations:
                     if result.mtimes is None:
                         result.mtimes = {}
                     result.mtimes.update(
                         {
-                            str(f): f.stat().st_mtime_ns
+                            str(f): os.stat(f, follow_symlinks=False).st_mtime_ns
                             for f in itertools.chain(
-                                *(iter_files(loc, "**/*.py") for loc in result.submodule_search_locations)
+                                *(Path(loc).rglob("**/*.py") for loc in result.submodule_search_locations)
                             )
                         }
                     )
@@ -987,16 +986,16 @@ class ImportsManager:
                     return None, import_name
 
                 if result.origin is not None:
-                    result.mtimes = {result.origin: Path(result.origin).stat().st_mtime_ns}
+                    result.mtimes = {result.origin: os.stat(result.origin, follow_symlinks=False).st_mtime_ns}
 
                 if result.submodule_search_locations:
                     if result.mtimes is None:
                         result.mtimes = {}
                     result.mtimes.update(
                         {
-                            str(f): f.stat().st_mtime_ns
+                            str(f): os.stat(f, follow_symlinks=False).st_mtime_ns
                             for f in itertools.chain(
-                                *(iter_files(loc, "**/*.py") for loc in result.submodule_search_locations)
+                                *(Path(loc).rglob("**/*.py") for loc in result.submodule_search_locations)
                             )
                         }
                     )
@@ -1015,7 +1014,10 @@ class ImportsManager:
         base_dir: str,
         variables: Optional[Dict[str, Any]] = None,
     ) -> str:
-        return self._library_files_cache.get(self._find_library, name, base_dir, variables)
+        if contains_variable(name, "$@&%"):
+            return self._library_files_cache.get(self._find_library, name, base_dir, variables)
+
+        return self._library_files_cache.get(self._find_library_simple, name, base_dir)
 
     def _find_library(
         self,
@@ -1023,17 +1025,19 @@ class ImportsManager:
         base_dir: str,
         variables: Optional[Dict[str, Any]] = None,
     ) -> str:
-        from robot.libraries import STDLIBS
-        from robot.variables.search import contains_variable
+        return find_library(
+            name,
+            str(self.root_folder),
+            base_dir,
+            self.get_resolvable_command_line_variables(),
+            variables,
+        )
 
-        if contains_variable(name, "$@&%"):
-            return find_library(
-                name,
-                str(self.root_folder),
-                base_dir,
-                self.get_resolvable_command_line_variables(),
-                variables,
-            )
+    def _find_library_simple(
+        self,
+        name: str,
+        base_dir: str,
+    ) -> str:
 
         if name in STDLIBS:
             result = ROBOT_LIBRARY_PACKAGE + "." + name
@@ -1052,7 +1056,10 @@ class ImportsManager:
         file_type: str = "Resource",
         variables: Optional[Dict[str, Any]] = None,
     ) -> str:
-        return self._resource_files_cache.get(self.__find_resource, name, base_dir, file_type, variables)
+        if contains_variable(name, "$@&%"):
+            return self._resource_files_cache.get(self.__find_resource, name, base_dir, file_type, variables)
+
+        return self._resource_files_cache.get(self.__find_resource_simple, name, base_dir, file_type)
 
     @_logger.call
     def __find_resource(
@@ -1062,64 +1069,71 @@ class ImportsManager:
         file_type: str = "Resource",
         variables: Optional[Dict[str, Any]] = None,
     ) -> str:
-        from robot.variables.search import contains_variable
+        return find_file(
+            name,
+            str(self.root_folder),
+            base_dir,
+            self.get_resolvable_command_line_variables(),
+            variables,
+            file_type,
+        )
 
-        if contains_variable(name, "$@&%"):
-            return find_file(
+    def __find_resource_simple(
+        self,
+        name: str,
+        base_dir: str,
+        file_type: str = "Resource",
+    ) -> str:
+        return find_file_ex(name, base_dir, file_type)
+
+    def find_variables(
+        self,
+        name: str,
+        base_dir: str,
+        variables: Optional[Dict[str, Any]] = None,
+        resolve_variables: bool = True,
+        resolve_command_line_vars: bool = True,
+    ) -> str:
+        if resolve_variables and contains_variable(name, "$@&%"):
+            return self._variables_files_cache.get(
+                self.__find_variables,
                 name,
-                str(self.root_folder),
                 base_dir,
-                self.get_resolvable_command_line_variables(),
                 variables,
-                file_type,
+                resolve_command_line_vars,
             )
+        return self._variables_files_cache.get(self.__find_variables_simple, name, base_dir)
 
-        return str(find_file_ex(name, base_dir, file_type))
-
-    def find_variables(
+    @_logger.call
+    def __find_variables(
         self,
         name: str,
         base_dir: str,
         variables: Optional[Dict[str, Any]] = None,
-        resolve_variables: bool = True,
         resolve_command_line_vars: bool = True,
     ) -> str:
-        return self._variables_files_cache.get(
-            self.__find_variables,
+        return find_variables(
             name,
+            str(self.root_folder),
             base_dir,
+            self.get_resolvable_command_line_variables() if resolve_command_line_vars else None,
             variables,
-            resolve_variables,
-            resolve_command_line_vars,
         )
 
     @_logger.call
-    def __find_variables(
+    def __find_variables_simple(
         self,
         name: str,
         base_dir: str,
-        variables: Optional[Dict[str, Any]] = None,
-        resolve_variables: bool = True,
-        resolve_command_line_vars: bool = True,
     ) -> str:
-        from robot.variables.search import contains_variable
-
-        if resolve_variables and contains_variable(name, "$@&%"):
-            return find_variables(
-                name,
-                str(self.root_folder),
-                base_dir,
-                self.get_resolvable_command_line_variables() if resolve_command_line_vars else None,
-                variables,
-            )
 
         if get_robot_version() >= (5, 0):
             if is_variables_by_path(name):
-                return str(find_file_ex(name, base_dir, "Variables"))
+                return find_file_ex(name, base_dir, "Variables")
 
             return name
 
-        return str(find_file_ex(name, base_dir, "Variables"))
+        return find_file_ex(name, base_dir, "Variables")
 
     @property
     def executor(self) -> ProcessPoolExecutor:
@@ -1155,29 +1169,26 @@ class ImportsManager:
 
         if meta is not None and not meta.has_errors:
 
-            meta_file = Path(self.lib_doc_cache_path, meta.filepath_base + ".meta.json")
-            if meta_file.exists():
+            meta_file = meta.filepath_base + ".meta"
+            if self.data_cache.cache_data_exists(CacheSection.LIBRARY, meta_file):
                 try:
                     spec_path = None
                     try:
-                        saved_meta = from_json(meta_file.read_text("utf-8"), LibraryMetaData)
+                        saved_meta = self.data_cache.read_cache_data(CacheSection.LIBRARY, meta_file, LibraryMetaData)
                         if saved_meta.has_errors:
                             self._logger.debug(
-                                lambda: "Saved library spec for {name}{args!r} is not used "
+                                lambda: f"Saved library spec for {name}{args!r} is not used "
                                 "due to errors in meta data",
                                 context_name="import",
                             )
 
                         if not saved_meta.has_errors and saved_meta == meta:
-                            spec_path = Path(
-                                self.lib_doc_cache_path,
-                                meta.filepath_base + ".spec.json",
-                            )
+                            spec_path = meta.filepath_base + ".spec"
 
                             self._logger.debug(
                                 lambda: f"Use cached library meta data for {name}", context_name="import"
                             )
-                            return from_json(spec_path.read_text("utf-8"), LibraryDoc)
+                            return self.data_cache.read_cache_data(CacheSection.LIBRARY, spec_path, LibraryDoc)
 
                     except (SystemExit, KeyboardInterrupt):
                         raise
@@ -1191,6 +1202,9 @@ class ImportsManager:
                     self._logger.exception(e)
 
         self._logger.debug(lambda: f"Load library in process {name}{args!r}", context_name="import")
+        # if self._process_pool_executor is None:
+        #     self._process_pool_executor = ProcessPoolExecutor(max_workers=1, mp_context=mp.get_context("spawn"))
+        # executor = self._process_pool_executor
         executor = ProcessPoolExecutor(max_workers=1, mp_context=mp.get_context("spawn"))
         try:
             try:
@@ -1222,19 +1236,17 @@ class ImportsManager:
             if meta is not None:
                 meta.has_errors = bool(result.errors)
 
-                meta_file = Path(self.lib_doc_cache_path, meta.filepath_base + ".meta.json")
-                spec_file = Path(self.lib_doc_cache_path, meta.filepath_base + ".spec.json")
-
-                spec_file.parent.mkdir(parents=True, exist_ok=True)
+                meta_file = meta.filepath_base + ".meta"
+                spec_file = meta.filepath_base + ".spec"
 
                 try:
-                    spec_file.write_text(as_json(result), "utf-8")
+                    self.data_cache.save_cache_data(CacheSection.LIBRARY, spec_file, result)
                 except (SystemExit, KeyboardInterrupt):
                     raise
                 except BaseException as e:
                     raise RuntimeError(f"Cannot write spec file for library '{name}' to '{spec_file}'") from e
 
-                meta_file.write_text(as_json(meta), "utf-8")
+                self.data_cache.save_cache_data(CacheSection.LIBRARY, meta_file, meta)
             else:
                 self._logger.debug(lambda: f"Skip caching library {name}{args!r}", context_name="import")
         except (SystemExit, KeyboardInterrupt):
@@ -1351,21 +1363,17 @@ class ImportsManager:
         )
 
         if meta is not None:
-            meta_file = Path(
-                self.variables_doc_cache_path,
-                meta.filepath_base + ".meta.json",
-            )
-            if meta_file.exists():
+            meta_file = meta.filepath_base + ".meta"
+
+            if self.data_cache.cache_data_exists(CacheSection.VARIABLES, meta_file):
                 try:
                     spec_path = None
                     try:
-                        saved_meta = from_json(meta_file.read_text("utf-8"), LibraryMetaData)
+                        saved_meta = self.data_cache.read_cache_data(CacheSection.VARIABLES, meta_file, LibraryMetaData)
                         if saved_meta == meta:
-                            spec_path = Path(
-                                self.variables_doc_cache_path,
-                                meta.filepath_base + ".spec.json",
-                            )
-                            return from_json(spec_path.read_text("utf-8"), VariablesDoc)
+                            spec_path = meta.filepath_base + ".spec"
+
+                            return self.data_cache.read_cache_data(CacheSection.VARIABLES, spec_path, VariablesDoc)
                     except (SystemExit, KeyboardInterrupt):
                         raise
                     except BaseException as e:
@@ -1406,23 +1414,16 @@ class ImportsManager:
 
         try:
             if meta is not None:
-                meta_file = Path(
-                    self.variables_doc_cache_path,
-                    meta.filepath_base + ".meta.json",
-                )
-                spec_file = Path(
-                    self.variables_doc_cache_path,
-                    meta.filepath_base + ".spec.json",
-                )
-                spec_file.parent.mkdir(parents=True, exist_ok=True)
+                meta_file = meta.filepath_base + ".meta"
+                spec_file = meta.filepath_base + ".spec"
 
                 try:
-                    spec_file.write_text(as_json(result), "utf-8")
+                    self.data_cache.save_cache_data(CacheSection.VARIABLES, spec_file, result)
                 except (SystemExit, KeyboardInterrupt):
                     raise
                 except BaseException as e:
                     raise RuntimeError(f"Cannot write spec file for variables '{name}' to '{spec_file}'") from e
-                meta_file.write_text(as_json(meta), "utf-8")
+                self.data_cache.save_cache_data(CacheSection.VARIABLES, meta_file, meta)
             else:
                 self._logger.debug(lambda: f"Skip caching variables {name}{args!r}", context_name="import")
         except (SystemExit, KeyboardInterrupt):